Welcome to mirror list, hosted at ThFree Co, Russian Federation.

gitlab.com/gitlab-org/gitlab-foss.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
authorGitLab Bot <gitlab-bot@gitlab.com>2023-05-17 19:05:49 +0300
committerGitLab Bot <gitlab-bot@gitlab.com>2023-05-17 19:05:49 +0300
commit43a25d93ebdabea52f99b05e15b06250cd8f07d7 (patch)
treedceebdc68925362117480a5d672bcff122fb625b /lib
parent20c84b99005abd1c82101dfeff264ac50d2df211 (diff)
Add latest changes from gitlab-org/gitlab@16-0-stable-eev16.0.0-rc42
Diffstat (limited to 'lib')
-rw-r--r--lib/api/access_requests.rb2
-rw-r--r--lib/api/admin/ci/variables.rb2
-rw-r--r--lib/api/admin/instance_clusters.rb2
-rw-r--r--lib/api/admin/plan_limits.rb1
-rw-r--r--lib/api/api.rb10
-rw-r--r--lib/api/applications.rb2
-rw-r--r--lib/api/branches.rb2
-rw-r--r--lib/api/bulk_imports.rb4
-rw-r--r--lib/api/ci/helpers/runner.rb18
-rw-r--r--lib/api/ci/jobs.rb10
-rw-r--r--lib/api/ci/pipeline_schedules.rb7
-rw-r--r--lib/api/ci/pipelines.rb20
-rw-r--r--lib/api/ci/runner.rb54
-rw-r--r--lib/api/ci/runners.rb6
-rw-r--r--lib/api/ci/variables.rb2
-rw-r--r--lib/api/clusters/agent_tokens.rb15
-rw-r--r--lib/api/clusters/agents.rb4
-rw-r--r--lib/api/commits.rb14
-rw-r--r--lib/api/composer_packages.rb2
-rw-r--r--lib/api/conan_instance_packages.rb6
-rw-r--r--lib/api/conan_project_packages.rb6
-rw-r--r--lib/api/concerns/packages/conan_endpoints.rb5
-rw-r--r--lib/api/concerns/packages/debian_package_endpoints.rb78
-rw-r--r--lib/api/concerns/packages/npm_endpoints.rb27
-rw-r--r--lib/api/debian_group_packages.rb7
-rw-r--r--lib/api/debian_project_packages.rb25
-rw-r--r--lib/api/deploy_keys.rb1
-rw-r--r--lib/api/draft_notes.rb104
-rw-r--r--lib/api/entities/application_setting.rb1
-rw-r--r--lib/api/entities/application_with_secret.rb8
-rw-r--r--lib/api/entities/ci/job_request/cache.rb2
-rw-r--r--lib/api/entities/ci/job_request/response.rb4
-rw-r--r--lib/api/entities/ci/pipeline_basic_with_metadata.rb13
-rw-r--r--lib/api/entities/ci/pipeline_with_metadata.rb13
-rw-r--r--lib/api/entities/ci/secure_file.rb2
-rw-r--r--lib/api/entities/clusters/agent_authorization.rb13
-rw-r--r--lib/api/entities/clusters/agents/authorizations/ci_access.rb17
-rw-r--r--lib/api/entities/internal/pages/lookup_path.rb9
-rw-r--r--lib/api/entities/merge_request_basic.rb11
-rw-r--r--lib/api/entities/ml/mlflow/run_info.rb2
-rw-r--r--lib/api/entities/note.rb1
-rw-r--r--lib/api/entities/plan_limit.rb1
-rw-r--r--lib/api/entities/project.rb53
-rw-r--r--lib/api/entities/project_job_token_scope.rb14
-rw-r--r--lib/api/entities/protected_ref_access.rb2
-rw-r--r--lib/api/entities/releases/link.rb1
-rw-r--r--lib/api/entities/user.rb17
-rw-r--r--lib/api/entities/user_preferences.rb2
-rw-r--r--lib/api/environments.rb11
-rw-r--r--lib/api/error_tracking/project_settings.rb40
-rw-r--r--lib/api/files.rb12
-rw-r--r--lib/api/generic_packages.rb4
-rw-r--r--lib/api/group_clusters.rb2
-rw-r--r--lib/api/group_container_repositories.rb2
-rw-r--r--lib/api/group_variables.rb2
-rw-r--r--lib/api/groups.rb6
-rw-r--r--lib/api/helpers.rb10
-rw-r--r--lib/api/helpers/integrations_helpers.rb66
-rw-r--r--lib/api/helpers/internal_helpers.rb1
-rw-r--r--lib/api/helpers/members_helpers.rb6
-rw-r--r--lib/api/helpers/merge_requests_helpers.rb3
-rw-r--r--lib/api/helpers/packages/conan/api_helpers.rb4
-rw-r--r--lib/api/helpers/packages/dependency_proxy_helpers.rb9
-rw-r--r--lib/api/helpers/packages/maven/basic_auth_helpers.rb21
-rw-r--r--lib/api/helpers/packages/npm.rb32
-rw-r--r--lib/api/helpers/packages_helpers.rb3
-rw-r--r--lib/api/helpers/projects_helpers.rb2
-rw-r--r--lib/api/import_github.rb2
-rw-r--r--lib/api/integrations.rb36
-rw-r--r--lib/api/integrations/slack/concerns/verifies_request.rb23
-rw-r--r--lib/api/integrations/slack/events.rb58
-rw-r--r--lib/api/integrations/slack/interactions.rb30
-rw-r--r--lib/api/integrations/slack/options.rb30
-rw-r--r--lib/api/integrations/slack/request.rb51
-rw-r--r--lib/api/internal/base.rb12
-rw-r--r--lib/api/internal/kubernetes.rb61
-rw-r--r--lib/api/internal/pages.rb21
-rw-r--r--lib/api/issues.rb2
-rw-r--r--lib/api/keys.rb2
-rw-r--r--lib/api/lint.rb19
-rw-r--r--lib/api/maven_packages.rb83
-rw-r--r--lib/api/members.rb2
-rw-r--r--lib/api/merge_requests.rb7
-rw-r--r--lib/api/metrics/dashboard/annotations.rb4
-rw-r--r--lib/api/metrics/user_starred_dashboards.rb4
-rw-r--r--lib/api/milestone_responses.rb45
-rw-r--r--lib/api/ml/mlflow.rb297
-rw-r--r--lib/api/ml/mlflow/api_helpers.rb54
-rw-r--r--lib/api/ml/mlflow/entrypoint.rb51
-rw-r--r--lib/api/ml/mlflow/experiments.rb80
-rw-r--r--lib/api/ml/mlflow/runs.rb152
-rw-r--r--lib/api/npm_project_packages.rb26
-rw-r--r--lib/api/nuget_project_packages.rb2
-rw-r--r--lib/api/package_files.rb2
-rw-r--r--lib/api/personal_access_tokens.rb26
-rw-r--r--lib/api/personal_access_tokens/self_information.rb2
-rw-r--r--lib/api/project_clusters.rb2
-rw-r--r--lib/api/project_container_repositories.rb10
-rw-r--r--lib/api/project_import.rb12
-rw-r--r--lib/api/project_job_token_scope.rb27
-rw-r--r--lib/api/project_templates.rb4
-rw-r--r--lib/api/projects.rb38
-rw-r--r--lib/api/protected_branches.rb18
-rw-r--r--lib/api/pypi_packages.rb2
-rw-r--r--lib/api/release/links.rb44
-rw-r--r--lib/api/releases.rb4
-rw-r--r--lib/api/repositories.rb4
-rw-r--r--lib/api/resource_access_tokens.rb34
-rw-r--r--lib/api/rpm_project_packages.rb1
-rw-r--r--lib/api/rubygem_packages.rb2
-rw-r--r--lib/api/search.rb3
-rw-r--r--lib/api/settings.rb5
-rw-r--r--lib/api/subscriptions.rb2
-rw-r--r--lib/api/tags.rb2
-rw-r--r--lib/api/templates.rb2
-rw-r--r--lib/api/terraform/modules/v1/packages.rb4
-rw-r--r--lib/api/terraform/state.rb41
-rw-r--r--lib/api/todos.rb2
-rw-r--r--lib/api/unleash.rb4
-rw-r--r--lib/api/users.rb196
-rw-r--r--lib/api/v3/github.rb20
-rw-r--r--lib/api/validations/validators/bulk_imports.rb46
-rw-r--r--lib/atlassian/jira_connect/serializers/branch_entity.rb5
-rw-r--r--lib/atlassian/jira_connect/serializers/build_entity.rb7
-rw-r--r--lib/atlassian/jira_connect/serializers/commit_entity.rb10
-rw-r--r--lib/atlassian/jira_issue_key_extractor.rb5
-rw-r--r--lib/atlassian/jira_issue_key_extractors/branch.rb42
-rw-r--r--lib/backup/database.rb47
-rw-r--r--lib/backup/gitaly_backup.rb28
-rw-r--r--lib/backup/manager.rb16
-rw-r--r--lib/backup/repositories.rb8
-rw-r--r--lib/banzai/filter/asset_proxy_filter.rb2
-rw-r--r--lib/banzai/filter/base_sanitization_filter.rb3
-rw-r--r--lib/banzai/filter/blockquote_fence_filter.rb4
-rw-r--r--lib/banzai/filter/code_language_filter.rb71
-rw-r--r--lib/banzai/filter/commit_trailers_filter.rb2
-rw-r--r--lib/banzai/filter/dollar_math_pre_filter.rb43
-rw-r--r--lib/banzai/filter/inline_embeds_filter.rb2
-rw-r--r--lib/banzai/filter/inline_observability_filter.rb45
-rw-r--r--lib/banzai/filter/issuable_reference_expansion_filter.rb43
-rw-r--r--lib/banzai/filter/kroki_filter.rb6
-rw-r--r--lib/banzai/filter/markdown_engines/base.rb23
-rw-r--r--lib/banzai/filter/markdown_engines/common_mark.rb18
-rw-r--r--lib/banzai/filter/markdown_filter.rb26
-rw-r--r--lib/banzai/filter/math_filter.rb2
-rw-r--r--lib/banzai/filter/mermaid_filter.rb2
-rw-r--r--lib/banzai/filter/plantuml_filter.rb2
-rw-r--r--lib/banzai/filter/reference_redactor_filter.rb4
-rw-r--r--lib/banzai/filter/references/abstract_reference_filter.rb10
-rw-r--r--lib/banzai/filter/references/commit_range_reference_filter.rb3
-rw-r--r--lib/banzai/filter/references/commit_reference_filter.rb14
-rw-r--r--lib/banzai/filter/references/design_reference_filter.rb2
-rw-r--r--lib/banzai/filter/references/issue_reference_filter.rb4
-rw-r--r--lib/banzai/filter/references/iteration_reference_filter.rb15
-rw-r--r--lib/banzai/filter/references/merge_request_reference_filter.rb3
-rw-r--r--lib/banzai/filter/references/snippet_reference_filter.rb3
-rw-r--r--lib/banzai/filter/references/user_reference_filter.rb6
-rw-r--r--lib/banzai/filter/references/work_item_reference_filter.rb26
-rw-r--r--lib/banzai/filter/repository_link_filter.rb2
-rw-r--r--lib/banzai/filter/syntax_highlight_filter.rb51
-rw-r--r--lib/banzai/filter/timeout_html_pipeline_filter.rb7
-rw-r--r--lib/banzai/filter/timeout_text_pipeline_filter.rb31
-rw-r--r--lib/banzai/issuable_extractor.rb4
-rw-r--r--lib/banzai/pipeline/ascii_doc_pipeline.rb1
-rw-r--r--lib/banzai/pipeline/gfm_pipeline.rb2
-rw-r--r--lib/banzai/pipeline/markup_pipeline.rb1
-rw-r--r--lib/banzai/reference_parser/commit_parser.rb21
-rw-r--r--lib/banzai/reference_parser/commit_range_parser.rb7
-rw-r--r--lib/banzai/reference_parser/issue_parser.rb31
-rw-r--r--lib/banzai/reference_parser/iteration_parser.rb22
-rw-r--r--lib/banzai/reference_parser/merge_request_parser.rb22
-rw-r--r--lib/banzai/reference_parser/user_parser.rb2
-rw-r--r--lib/banzai/reference_parser/work_item_parser.rb17
-rw-r--r--lib/banzai/render_context.rb5
-rw-r--r--lib/banzai/renderer.rb42
-rw-r--r--lib/bulk_imports/clients/graphql.rb16
-rw-r--r--lib/bulk_imports/clients/http.rb11
-rw-r--r--lib/bulk_imports/error.rb26
-rw-r--r--lib/bulk_imports/features.rb15
-rw-r--r--lib/bulk_imports/groups/stage.rb19
-rw-r--r--lib/bulk_imports/groups/transformers/group_attributes_transformer.rb24
-rw-r--r--lib/bulk_imports/ndjson_pipeline.rb43
-rw-r--r--lib/bulk_imports/path_normalization.rb32
-rw-r--r--lib/bulk_imports/projects/graphql/get_project_query.rb1
-rw-r--r--lib/bulk_imports/projects/pipelines/commit_notes_pipeline.rb15
-rw-r--r--lib/bulk_imports/projects/pipelines/references_pipeline.rb2
-rw-r--r--lib/bulk_imports/projects/stage.rb5
-rw-r--r--lib/bulk_imports/projects/transformers/project_attributes_transformer.rb9
-rw-r--r--lib/bulk_imports/uniquify.rb24
-rw-r--r--lib/container_registry/gitlab_api_client.rb75
-rw-r--r--lib/error_tracking/sentry_client/token.rb13
-rw-r--r--lib/feature.rb21
-rw-r--r--lib/feature/gitaly.rb12
-rw-r--r--lib/feature/logger.rb2
-rw-r--r--lib/feature_groups/gitlab_team_members.rb31
-rw-r--r--lib/generators/batched_background_migration/USAGE12
-rw-r--r--lib/generators/batched_background_migration/batched_background_migration_generator.rb73
-rw-r--r--lib/generators/batched_background_migration/templates/batched_background_migration_dictionary.template6
-rw-r--r--lib/generators/batched_background_migration/templates/batched_background_migration_job.template22
-rw-r--r--lib/generators/batched_background_migration/templates/batched_background_migration_job_spec.template7
-rw-r--r--lib/generators/batched_background_migration/templates/queue_batched_background_migration.template28
-rw-r--r--lib/generators/batched_background_migration/templates/queue_batched_background_migration_spec.template26
-rw-r--r--lib/generators/gitlab/snowplow_event_definition_generator.rb13
-rw-r--r--lib/gitlab.rb4
-rw-r--r--lib/gitlab/access.rb1
-rw-r--r--lib/gitlab/action_cable/request_store_callbacks.rb2
-rw-r--r--lib/gitlab/analytics/cycle_analytics/aggregated/records_fetcher.rb8
-rw-r--r--lib/gitlab/analytics/cycle_analytics/aggregated/stage_query_helpers.rb2
-rw-r--r--lib/gitlab/analytics/cycle_analytics/average.rb2
-rw-r--r--lib/gitlab/analytics/cycle_analytics/median.rb3
-rw-r--r--lib/gitlab/analytics/cycle_analytics/records_fetcher.rb8
-rw-r--r--lib/gitlab/analytics/cycle_analytics/request_params.rb103
-rw-r--r--lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start.rb2
-rw-r--r--lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb4
-rw-r--r--lib/gitlab/app_logger.rb10
-rw-r--r--lib/gitlab/application_context.rb7
-rw-r--r--lib/gitlab/application_rate_limiter.rb8
-rw-r--r--lib/gitlab/audit/auditor.rb10
-rw-r--r--lib/gitlab/auth.rb51
-rw-r--r--lib/gitlab/auth/auth_finders.rb6
-rw-r--r--lib/gitlab/auth/ldap/config.rb4
-rw-r--r--lib/gitlab/auth/o_auth/auth_hash.rb8
-rw-r--r--lib/gitlab/auth/o_auth/session.rb23
-rw-r--r--lib/gitlab/auth/o_auth/user.rb4
-rw-r--r--lib/gitlab/auth/otp/duo_auth.rb13
-rw-r--r--lib/gitlab/auth/otp/strategies/duo_auth/manual_otp.rb46
-rw-r--r--lib/gitlab/auth/u2f_webauthn_converter.rb40
-rw-r--r--lib/gitlab/auth/user_access_denied_reason.rb3
-rw-r--r--lib/gitlab/avatar_cache.rb8
-rw-r--r--lib/gitlab/background_migration/backfill_admin_mode_scope_for_personal_access_tokens.rb8
-rw-r--r--lib/gitlab/background_migration/backfill_compliance_violations.rb17
-rw-r--r--lib/gitlab/background_migration/backfill_design_management_repositories.rb29
-rw-r--r--lib/gitlab/background_migration/backfill_integrations_enable_ssl_verification.rb14
-rw-r--r--lib/gitlab/background_migration/backfill_namespace_ldap_settings.rb17
-rw-r--r--lib/gitlab/background_migration/backfill_namespace_traversal_ids_children.rb76
-rw-r--r--lib/gitlab/background_migration/backfill_namespace_traversal_ids_roots.rb51
-rw-r--r--lib/gitlab/background_migration/backfill_partitioned_table.rb43
-rw-r--r--lib/gitlab/background_migration/backfill_prepared_at_merge_requests.rb18
-rw-r--r--lib/gitlab/background_migration/backfill_project_wiki_repositories.rb35
-rw-r--r--lib/gitlab/background_migration/backfill_upvotes_count_on_issues.rb40
-rw-r--r--lib/gitlab/background_migration/backfill_user_namespace.rb38
-rw-r--r--lib/gitlab/background_migration/backfill_work_item_type_id_for_issues.rb6
-rw-r--r--lib/gitlab/background_migration/batched_migration_job.rb15
-rw-r--r--lib/gitlab/background_migration/cleanup_orphaned_lfs_objects_projects.rb78
-rw-r--r--lib/gitlab/background_migration/cleanup_personal_access_tokens_with_nil_expires_at.rb22
-rw-r--r--lib/gitlab/background_migration/create_vulnerability_links.rb14
-rw-r--r--lib/gitlab/background_migration/delete_orphaned_deployments.rb32
-rw-r--r--lib/gitlab/background_migration/delete_orphaned_packages_dependencies.rb27
-rw-r--r--lib/gitlab/background_migration/disable_expiration_policies_linked_to_no_container_images.rb41
-rw-r--r--lib/gitlab/background_migration/disable_legacy_open_source_license_for_no_issues_no_repo_projects.rb5
-rw-r--r--lib/gitlab/background_migration/disable_legacy_open_source_license_for_one_member_no_repo_projects.rb5
-rw-r--r--lib/gitlab/background_migration/drop_invalid_remediations.rb14
-rw-r--r--lib/gitlab/background_migration/drop_invalid_security_findings.rb47
-rw-r--r--lib/gitlab/background_migration/drop_invalid_vulnerabilities.rb37
-rw-r--r--lib/gitlab/background_migration/encrypt_ci_trigger_token.rb3
-rw-r--r--lib/gitlab/background_migration/encrypt_integration_properties.rb16
-rw-r--r--lib/gitlab/background_migration/extract_project_topics_into_separate_table.rb63
-rw-r--r--lib/gitlab/background_migration/fix_incoherent_packages_size_on_project_statistics.rb4
-rw-r--r--lib/gitlab/background_migration/fix_merge_request_diff_commit_users.rb21
-rw-r--r--lib/gitlab/background_migration/fix_vulnerability_reads_has_issues.rb33
-rw-r--r--lib/gitlab/background_migration/issues_internal_id_scope_updater.rb66
-rw-r--r--lib/gitlab/background_migration/logger.rb2
-rw-r--r--lib/gitlab/background_migration/migrate_evidences_for_vulnerability_findings.rb76
-rw-r--r--lib/gitlab/background_migration/migrate_human_user_type.rb37
-rw-r--r--lib/gitlab/background_migration/migrate_links_for_vulnerability_findings.rb92
-rw-r--r--lib/gitlab/background_migration/migrate_merge_request_diff_commit_users.rb296
-rw-r--r--lib/gitlab/background_migration/migrate_project_taggings_context_from_tags_to_topics.rb21
-rw-r--r--lib/gitlab/background_migration/migrate_remediations_for_vulnerability_findings.rb164
-rw-r--r--lib/gitlab/background_migration/migrate_shared_vulnerability_identifiers.rb18
-rw-r--r--lib/gitlab/background_migration/migrate_u2f_webauthn.rb28
-rw-r--r--lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature.rb52
-rw-r--r--lib/gitlab/background_migration/populate_topics_total_projects_count_cache.rb29
-rw-r--r--lib/gitlab/background_migration/populate_uuids_for_security_findings.rb18
-rw-r--r--lib/gitlab/background_migration/populate_vulnerability_dismissal_fields.rb90
-rw-r--r--lib/gitlab/background_migration/remove_duplicate_vulnerabilities_findings.rb64
-rw-r--r--lib/gitlab/background_migration/remove_occurrence_pipelines_and_duplicate_vulnerabilities_findings.rb2
-rw-r--r--lib/gitlab/background_migration/remove_project_group_link_with_missing_groups.rb31
-rw-r--r--lib/gitlab/background_migration/reset_status_on_container_repositories.rb10
-rw-r--r--lib/gitlab/background_migration/steal_migrate_merge_request_diff_commit_users.rb33
-rw-r--r--lib/gitlab/background_migration/update_timelogs_project_id.rb44
-rw-r--r--lib/gitlab/background_migration/update_users_where_two_factor_auth_required_from_group.rb129
-rw-r--r--lib/gitlab/background_migration/update_vulnerability_occurrences_location.rb14
-rw-r--r--lib/gitlab/backup_logger.rb2
-rw-r--r--lib/gitlab/bare_repository_import/importer.rb132
-rw-r--r--lib/gitlab/bare_repository_import/repository.rb72
-rw-r--r--lib/gitlab/bitbucket_import/importer.rb2
-rw-r--r--lib/gitlab/bullet/exclusions.rb3
-rw-r--r--lib/gitlab/cache/client.rb65
-rw-r--r--lib/gitlab/cache/metadata.rb14
-rw-r--r--lib/gitlab/cache/metrics.rb1
-rw-r--r--lib/gitlab/changes_list.rb4
-rw-r--r--lib/gitlab/chat/responder.rb18
-rw-r--r--lib/gitlab/checks/base_single_checker.rb2
-rw-r--r--lib/gitlab/checks/changes_access.rb14
-rw-r--r--lib/gitlab/checks/diff_check.rb4
-rw-r--r--lib/gitlab/checks/matching_merge_request.rb2
-rw-r--r--lib/gitlab/checks/single_change_access.rb10
-rw-r--r--lib/gitlab/ci/ansi2json/parser.rb4
-rw-r--r--lib/gitlab/ci/ansi2json/state.rb65
-rw-r--r--lib/gitlab/ci/badge/release/latest_release.rb3
-rw-r--r--lib/gitlab/ci/badge/release/template.rb8
-rw-r--r--lib/gitlab/ci/build/cache.rb18
-rw-r--r--lib/gitlab/ci/build/rules.rb11
-rw-r--r--lib/gitlab/ci/components/instance_path.rb10
-rw-r--r--lib/gitlab/ci/config.rb14
-rw-r--r--lib/gitlab/ci/config/entry/cache.rb11
-rw-r--r--lib/gitlab/ci/config/entry/job.rb23
-rw-r--r--lib/gitlab/ci/config/entry/product/parallel.rb2
-rw-r--r--lib/gitlab/ci/config/entry/publish.rb24
-rw-r--r--lib/gitlab/ci/config/entry/rules/rule.rb10
-rw-r--r--lib/gitlab/ci/config/external/context.rb19
-rw-r--r--lib/gitlab/ci/config/external/file/artifact.rb2
-rw-r--r--lib/gitlab/ci/config/external/file/base.rb82
-rw-r--r--lib/gitlab/ci/config/external/file/component.rb7
-rw-r--r--lib/gitlab/ci/config/external/file/project.rb78
-rw-r--r--lib/gitlab/ci/config/external/interpolator.rb127
-rw-r--r--lib/gitlab/ci/config/external/mapper/matcher.rb39
-rw-r--r--lib/gitlab/ci/config/external/mapper/verifier.rb51
-rw-r--r--lib/gitlab/ci/config/header/input.rb25
-rw-r--r--lib/gitlab/ci/config/header/root.rb36
-rw-r--r--lib/gitlab/ci/config/header/spec.rb24
-rw-r--r--lib/gitlab/ci/config/yaml.rb47
-rw-r--r--lib/gitlab/ci/config/yaml/result.rb40
-rw-r--r--lib/gitlab/ci/input/arguments/base.rb62
-rw-r--r--lib/gitlab/ci/input/arguments/default.rb48
-rw-r--r--lib/gitlab/ci/input/arguments/options.rb55
-rw-r--r--lib/gitlab/ci/input/arguments/required.rb55
-rw-r--r--lib/gitlab/ci/input/arguments/unknown.rb31
-rw-r--r--lib/gitlab/ci/input/inputs.rb73
-rw-r--r--lib/gitlab/ci/interpolation/access.rb6
-rw-r--r--lib/gitlab/ci/interpolation/context.rb6
-rw-r--r--lib/gitlab/ci/jwt.rb30
-rw-r--r--lib/gitlab/ci/jwt_v2.rb32
-rw-r--r--lib/gitlab/ci/parsers/security/common.rb1
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schema_validator.rb39
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/14.0.0/container-scanning-report-format.json741
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/14.0.0/coverage-fuzzing-report-format.json711
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/14.0.0/dast-report-format.json1128
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/14.0.0/dependency-scanning-report-format.json805
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/14.0.0/sast-report-format.json706
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/14.0.0/secret-detection-report-format.json729
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/14.0.1/container-scanning-report-format.json809
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/14.0.1/coverage-fuzzing-report-format.json779
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/14.0.1/dast-report-format.json1196
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/14.0.1/dependency-scanning-report-format.json873
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/14.0.1/sast-report-format.json774
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/14.0.1/secret-detection-report-format.json797
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/14.0.2/container-scanning-report-format.json871
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/14.0.2/coverage-fuzzing-report-format.json841
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/14.0.2/dast-report-format.json1258
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/14.0.2/dependency-scanning-report-format.json935
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/14.0.2/sast-report-format.json836
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/14.0.2/secret-detection-report-format.json859
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/14.0.3/container-scanning-report-format.json904
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/14.0.3/coverage-fuzzing-report-format.json874
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/14.0.3/dast-report-format.json1291
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/14.0.3/dependency-scanning-report-format.json968
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/14.0.3/sast-report-format.json869
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/14.0.3/secret-detection-report-format.json892
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/14.0.4/cluster-image-scanning-report-format.json977
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/14.0.4/container-scanning-report-format.json904
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/14.0.4/coverage-fuzzing-report-format.json874
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/14.0.4/dast-report-format.json1291
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/14.0.4/dependency-scanning-report-format.json968
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/14.0.4/sast-report-format.json869
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/14.0.4/secret-detection-report-format.json892
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/14.0.5/cluster-image-scanning-report-format.json977
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/14.0.5/container-scanning-report-format.json910
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/14.0.5/coverage-fuzzing-report-format.json874
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/14.0.5/dast-report-format.json1291
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/14.0.5/dependency-scanning-report-format.json968
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/14.0.5/sast-report-format.json869
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/14.0.5/secret-detection-report-format.json892
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/14.0.6/cluster-image-scanning-report-format.json977
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/14.0.6/coverage-fuzzing-report-format.json874
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/14.0.6/dependency-scanning-report-format.json968
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/14.0.6/sast-report-format.json869
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/14.0.6/secret-detection-report-format.json892
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/14.1.0/cluster-image-scanning-report-format.json977
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/14.1.0/container-scanning-report-format.json911
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/14.1.0/dast-report-format.json1291
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/container-scanning-report-format.json911
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/coverage-fuzzing-report-format.json874
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/dast-report-format.json1291
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/dependency-scanning-report-format.json968
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/sast-report-format.json869
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/secret-detection-report-format.json892
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/cluster-image-scanning-report-format.json977
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/container-scanning-report-format.json911
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/coverage-fuzzing-report-format.json874
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/dast-report-format.json1287
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/dependency-scanning-report-format.json968
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/sast-report-format.json869
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/secret-detection-report-format.json892
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/cluster-image-scanning-report-format.json977
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/container-scanning-report-format.json911
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/coverage-fuzzing-report-format.json874
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/dast-report-format.json1287
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/dependency-scanning-report-format.json968
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/sast-report-format.json869
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/secret-detection-report-format.json892
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/15.0.6/cluster-image-scanning-report-format.json (renamed from lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/cluster-image-scanning-report-format.json)178
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/15.0.6/container-scanning-report-format.json (renamed from lib/gitlab/ci/parsers/security/validators/schemas/14.0.6/container-scanning-report-format.json)179
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/15.0.6/coverage-fuzzing-report-format.json (renamed from lib/gitlab/ci/parsers/security/validators/schemas/14.1.0/coverage-fuzzing-report-format.json)171
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/15.0.6/dast-report-format.json (renamed from lib/gitlab/ci/parsers/security/validators/schemas/14.0.6/dast-report-format.json)183
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/15.0.6/dependency-scanning-report-format.json (renamed from lib/gitlab/ci/parsers/security/validators/schemas/14.1.0/dependency-scanning-report-format.json)185
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/15.0.6/sast-report-format.json (renamed from lib/gitlab/ci/parsers/security/validators/schemas/14.1.0/sast-report-format.json)171
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/15.0.6/secret-detection-report-format.json (renamed from lib/gitlab/ci/parsers/security/validators/schemas/14.1.0/secret-detection-report-format.json)172
-rw-r--r--lib/gitlab/ci/pipeline/chain/command.rb3
-rw-r--r--lib/gitlab/ci/pipeline/chain/config/content.rb1
-rw-r--r--lib/gitlab/ci/pipeline/chain/config/process.rb1
-rw-r--r--lib/gitlab/ci/pipeline/chain/limit/activity.rb23
-rw-r--r--lib/gitlab/ci/pipeline/duration.rb25
-rw-r--r--lib/gitlab/ci/pipeline/seed/build.rb1
-rw-r--r--lib/gitlab/ci/pipeline/seed/build/cache.rb4
-rw-r--r--lib/gitlab/ci/project_config.rb1
-rw-r--r--lib/gitlab/ci/project_config/auto_devops.rb4
-rw-r--r--lib/gitlab/ci/project_config/external_project.rb4
-rw-r--r--lib/gitlab/ci/project_config/remote.rb4
-rw-r--r--lib/gitlab/ci/project_config/repository.rb4
-rw-r--r--lib/gitlab/ci/project_config/source.rb5
-rw-r--r--lib/gitlab/ci/reports/security/finding.rb8
-rw-r--r--lib/gitlab/ci/reports/security/report.rb5
-rw-r--r--lib/gitlab/ci/reports/security/vulnerability_reports_comparer.rb165
-rw-r--r--lib/gitlab/ci/resource_groups/logger.rb13
-rw-r--r--lib/gitlab/ci/runner_releases.rb5
-rw-r--r--lib/gitlab/ci/secure_files/cer.rb2
-rw-r--r--lib/gitlab/ci/secure_files/p12.rb2
-rw-r--r--lib/gitlab/ci/status/build/erased.rb4
-rw-r--r--lib/gitlab/ci/status/build/factory.rb4
-rw-r--r--lib/gitlab/ci/status/composite.rb56
-rw-r--r--lib/gitlab/ci/status/processable/waiting_for_resource.rb30
-rw-r--r--lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml1
-rw-r--r--lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Jobs/Container-Scanning.gitlab-ci.yml7
-rw-r--r--lib/gitlab/ci/templates/Jobs/Container-Scanning.latest.gitlab-ci.yml7
-rw-r--r--lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml48
-rw-r--r--lib/gitlab/ci/templates/Jobs/Dependency-Scanning.latest.gitlab-ci.yml48
-rw-r--r--lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Jobs/License-Scanning.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Jobs/License-Scanning.latest.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Jobs/SAST-IaC.gitlab-ci.yml4
-rw-r--r--lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml4
-rw-r--r--lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml62
-rw-r--r--lib/gitlab/ci/templates/Jobs/SAST.latest.gitlab-ci.yml66
-rw-r--r--lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml4
-rw-r--r--lib/gitlab/ci/templates/Jobs/Secret-Detection.latest.gitlab-ci.yml4
-rw-r--r--lib/gitlab/ci/templates/Python.gitlab-ci.yml14
-rw-r--r--lib/gitlab/ci/templates/Security/API-Discovery.gitlab-ci.yml66
-rw-r--r--lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml9
-rw-r--r--lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml9
-rw-r--r--lib/gitlab/ci/templates/Security/BAS.latest.gitlab-ci.yml65
-rw-r--r--lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Security/Coverage-Fuzzing.latest.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml9
-rw-r--r--lib/gitlab/ci/templates/Security/DAST-API.latest.gitlab-ci.yml9
-rw-r--r--lib/gitlab/ci/templates/Security/DAST-On-Demand-API-Scan.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Security/DAST-On-Demand-Scan.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml18
-rw-r--r--lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml14
-rw-r--r--lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Terraform.gitlab-ci.yml4
-rw-r--r--lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml11
-rw-r--r--lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml8
-rw-r--r--lib/gitlab/ci/templates/Terraform/Module-Base.gitlab-ci.yml4
-rw-r--r--lib/gitlab/ci/templates/Verify/Load-Performance-Testing.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/dotNET.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/trace/chunked_io.rb7
-rw-r--r--lib/gitlab/ci/variables/builder.rb12
-rw-r--r--lib/gitlab/ci/variables/builder/pipeline.rb10
-rw-r--r--lib/gitlab/ci/yaml_processor.rb31
-rw-r--r--lib/gitlab/ci/yaml_processor/result.rb3
-rw-r--r--lib/gitlab/color.rb39
-rw-r--r--lib/gitlab/color_schemes.rb2
-rw-r--r--lib/gitlab/config/entry/validators.rb2
-rw-r--r--lib/gitlab/config/loader/multi_doc_yaml.rb69
-rw-r--r--lib/gitlab/config/loader/yaml.rb8
-rw-r--r--lib/gitlab/config_checker/external_database_checker.rb6
-rw-r--r--lib/gitlab/consul/internal.rb2
-rw-r--r--lib/gitlab/content_security_policy/config_loader.rb2
-rw-r--r--lib/gitlab/cycle_analytics/stage_summary.rb2
-rw-r--r--lib/gitlab/data_builder/deployment.rb1
-rw-r--r--lib/gitlab/data_builder/pipeline.rb7
-rw-r--r--lib/gitlab/database.rb31
-rw-r--r--lib/gitlab/database/async_constraints.rb (renamed from lib/gitlab/database/async_foreign_keys.rb)6
-rw-r--r--lib/gitlab/database/async_constraints/migration_helpers.rb (renamed from lib/gitlab/database/async_foreign_keys/migration_helpers.rb)59
-rw-r--r--lib/gitlab/database/async_constraints/postgres_async_constraint_validation.rb35
-rw-r--r--lib/gitlab/database/async_constraints/validators.rb20
-rw-r--r--lib/gitlab/database/async_constraints/validators/base.rb91
-rw-r--r--lib/gitlab/database/async_constraints/validators/check_constraint.rb19
-rw-r--r--lib/gitlab/database/async_constraints/validators/foreign_key.rb21
-rw-r--r--lib/gitlab/database/async_foreign_keys/foreign_key_validator.rb94
-rw-r--r--lib/gitlab/database/async_foreign_keys/postgres_async_foreign_key_validation.rb21
-rw-r--r--lib/gitlab/database/async_indexes/migration_helpers.rb54
-rw-r--r--lib/gitlab/database/background_migration/batch_optimizer.rb13
-rw-r--r--lib/gitlab/database/background_migration/batched_job.rb81
-rw-r--r--lib/gitlab/database/background_migration/batched_migration.rb5
-rw-r--r--lib/gitlab/database/background_migration/batched_migration_wrapper.rb13
-rw-r--r--lib/gitlab/database/background_migration/health_status.rb14
-rw-r--r--lib/gitlab/database/background_migration/health_status/indicators/patroni_apdex.rb90
-rw-r--r--lib/gitlab/database/background_migration/health_status/signals.rb2
-rw-r--r--lib/gitlab/database/background_migration/sub_batch_timeout_error.rb17
-rw-r--r--lib/gitlab/database/background_migration_job.rb4
-rw-r--r--lib/gitlab/database/batch_count.rb2
-rw-r--r--lib/gitlab/database/dynamic_model_helpers.rb20
-rw-r--r--lib/gitlab/database/gitlab_schema.rb49
-rw-r--r--lib/gitlab/database/load_balancing/action_cable_callbacks.rb4
-rw-r--r--lib/gitlab/database/load_balancing/connection_proxy.rb7
-rw-r--r--lib/gitlab/database/load_balancing/logger.rb2
-rw-r--r--lib/gitlab/database/load_balancing/service_discovery.rb9
-rw-r--r--lib/gitlab/database/load_balancing/sidekiq_client_middleware.rb30
-rw-r--r--lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb45
-rw-r--r--lib/gitlab/database/load_balancing/wal_tracking_receiver.rb23
-rw-r--r--lib/gitlab/database/load_balancing/wal_tracking_sender.rb30
-rw-r--r--lib/gitlab/database/lock_writes_manager.rb14
-rw-r--r--lib/gitlab/database/migration.rb1
-rw-r--r--lib/gitlab/database/migration_helpers.rb154
-rw-r--r--lib/gitlab/database/migration_helpers/automatic_lock_writes_on_tables.rb11
-rw-r--r--lib/gitlab/database/migration_helpers/convert_to_bigint.rb21
-rw-r--r--lib/gitlab/database/migration_helpers/loose_foreign_key_helpers.rb14
-rw-r--r--lib/gitlab/database/migration_helpers/v2.rb40
-rw-r--r--lib/gitlab/database/migration_helpers/wraparound_vacuum_helpers.rb87
-rw-r--r--lib/gitlab/database/migrations/batched_background_migration_helpers.rb11
-rw-r--r--lib/gitlab/database/migrations/constraints_helpers.rb40
-rw-r--r--lib/gitlab/database/migrations/instrumentation.rb2
-rw-r--r--lib/gitlab/database/migrations/observation.rb2
-rw-r--r--lib/gitlab/database/migrations/pg_backend_pid.rb40
-rw-r--r--lib/gitlab/database/migrations/runner_backoff/active_record_mixin.rb34
-rw-r--r--lib/gitlab/database/migrations/runner_backoff/communicator.rb66
-rw-r--r--lib/gitlab/database/migrations/runner_backoff/migration_helpers.rb25
-rw-r--r--lib/gitlab/database/migrations/sidekiq_helpers.rb7
-rw-r--r--lib/gitlab/database/migrations/test_batched_background_runner.rb2
-rw-r--r--lib/gitlab/database/obsolete_ignored_columns.rb6
-rw-r--r--lib/gitlab/database/partitioning.rb5
-rw-r--r--lib/gitlab/database/partitioning/ci_sliding_list_strategy.rb40
-rw-r--r--lib/gitlab/database/partitioning/convert_table_to_first_list_partition.rb268
-rw-r--r--lib/gitlab/database/partitioning/list/convert_table.rb317
-rw-r--r--lib/gitlab/database/partitioning/list/locking_configuration.rb65
-rw-r--r--lib/gitlab/database/partitioning/partition_manager.rb2
-rw-r--r--lib/gitlab/database/partitioning/replace_table.rb2
-rw-r--r--lib/gitlab/database/partitioning_migration_helpers/backfill_partitioned_table.rb37
-rw-r--r--lib/gitlab/database/partitioning_migration_helpers/bulk_copy.rb42
-rw-r--r--lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers.rb71
-rw-r--r--lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb103
-rw-r--r--lib/gitlab/database/pg_depend.rb19
-rw-r--r--lib/gitlab/database/postgres_foreign_key.rb10
-rw-r--r--lib/gitlab/database/postgres_partition.rb10
-rw-r--r--lib/gitlab/database/query_analyzers/gitlab_schemas_metrics.rb2
-rw-r--r--lib/gitlab/database/query_analyzers/gitlab_schemas_validate_connection.rb2
-rw-r--r--lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb28
-rw-r--r--lib/gitlab/database/query_analyzers/restrict_allowed_schemas.rb2
-rw-r--r--lib/gitlab/database/reindexing.rb2
-rw-r--r--lib/gitlab/database/schema_helpers.rb6
-rw-r--r--lib/gitlab/database/schema_validation/adapters/column_database_adapter.rb47
-rw-r--r--lib/gitlab/database/schema_validation/adapters/column_structure_sql_adapter.rb128
-rw-r--r--lib/gitlab/database/schema_validation/database.rb94
-rw-r--r--lib/gitlab/database/schema_validation/inconsistency.rb65
-rw-r--r--lib/gitlab/database/schema_validation/inconsistency_filter.rb43
-rw-r--r--lib/gitlab/database/schema_validation/index.rb25
-rw-r--r--lib/gitlab/database/schema_validation/indexes.rb37
-rw-r--r--lib/gitlab/database/schema_validation/pg_types.rb73
-rw-r--r--lib/gitlab/database/schema_validation/runner.rb23
-rw-r--r--lib/gitlab/database/schema_validation/schema_inconsistency.rb15
-rw-r--r--lib/gitlab/database/schema_validation/schema_objects/base.rb31
-rw-r--r--lib/gitlab/database/schema_validation/schema_objects/column.rb23
-rw-r--r--lib/gitlab/database/schema_validation/schema_objects/index.rb15
-rw-r--r--lib/gitlab/database/schema_validation/schema_objects/table.rb40
-rw-r--r--lib/gitlab/database/schema_validation/schema_objects/trigger.rb15
-rw-r--r--lib/gitlab/database/schema_validation/structure_sql.rb65
-rw-r--r--lib/gitlab/database/schema_validation/track_inconsistency.rb98
-rw-r--r--lib/gitlab/database/schema_validation/validators/base_validator.rb46
-rw-r--r--lib/gitlab/database/schema_validation/validators/different_definition_indexes.rb24
-rw-r--r--lib/gitlab/database/schema_validation/validators/different_definition_tables.rb50
-rw-r--r--lib/gitlab/database/schema_validation/validators/different_definition_triggers.rb24
-rw-r--r--lib/gitlab/database/schema_validation/validators/extra_indexes.rb21
-rw-r--r--lib/gitlab/database/schema_validation/validators/extra_table_columns.rb32
-rw-r--r--lib/gitlab/database/schema_validation/validators/extra_tables.rb21
-rw-r--r--lib/gitlab/database/schema_validation/validators/extra_triggers.rb21
-rw-r--r--lib/gitlab/database/schema_validation/validators/missing_indexes.rb21
-rw-r--r--lib/gitlab/database/schema_validation/validators/missing_table_columns.rb32
-rw-r--r--lib/gitlab/database/schema_validation/validators/missing_tables.rb21
-rw-r--r--lib/gitlab/database/schema_validation/validators/missing_triggers.rb21
-rw-r--r--lib/gitlab/database/tables_locker.rb41
-rw-r--r--lib/gitlab/database_importers/instance_administrators/create_group.rb133
-rw-r--r--lib/gitlab/database_importers/security/training_providers/importer.rb9
-rw-r--r--lib/gitlab/database_importers/self_monitoring/helpers.rb25
-rw-r--r--lib/gitlab/database_importers/self_monitoring/project/create_service.rb171
-rw-r--r--lib/gitlab/database_importers/self_monitoring/project/delete_service.rb46
-rw-r--r--lib/gitlab/database_importers/work_items/base_type_importer.rb41
-rw-r--r--lib/gitlab/deprecation_json_logger.rb2
-rw-r--r--lib/gitlab/design_management/copy_design_collection_model_attributes.yml3
-rw-r--r--lib/gitlab/diff/highlight.rb20
-rw-r--r--lib/gitlab/diff/highlight_cache.rb1
-rw-r--r--lib/gitlab/diff/inline_diff.rb21
-rw-r--r--lib/gitlab/diff/parser.rb2
-rw-r--r--lib/gitlab/discussions_diff/highlight_cache.rb20
-rw-r--r--lib/gitlab/doorkeeper_secret_storing/secret/pbkdf2_sha512.rb7
-rw-r--r--lib/gitlab/email/handler/create_note_handler.rb4
-rw-r--r--lib/gitlab/email/hook/silent_mode_interceptor.rb27
-rw-r--r--lib/gitlab/email/hook/validate_addresses_interceptor.rb32
-rw-r--r--lib/gitlab/email/html_parser.rb6
-rw-r--r--lib/gitlab/email/html_to_markdown_parser.rb35
-rw-r--r--lib/gitlab/email/incoming_email.rb36
-rw-r--r--lib/gitlab/email/receiver.rb6
-rw-r--r--lib/gitlab/email/service_desk_email.rb28
-rw-r--r--lib/gitlab/email/service_desk_receiver.rb2
-rw-r--r--lib/gitlab/emoji.rb14
-rw-r--r--lib/gitlab/encrypted_incoming_email_command.rb2
-rw-r--r--lib/gitlab/encrypted_service_desk_email_command.rb2
-rw-r--r--lib/gitlab/etag_caching/router/graphql.rb7
-rw-r--r--lib/gitlab/event_store.rb4
-rw-r--r--lib/gitlab/exception_log_formatter.rb4
-rw-r--r--lib/gitlab/favicon.rb11
-rw-r--r--lib/gitlab/file_finder.rb14
-rw-r--r--lib/gitlab/git.rb17
-rw-r--r--lib/gitlab/git/blame_mode.rb31
-rw-r--r--lib/gitlab/git/blame_pagination.rb78
-rw-r--r--lib/gitlab/git/commit.rb6
-rw-r--r--lib/gitlab/git/diff_collection.rb2
-rw-r--r--lib/gitlab/git/ref.rb2
-rw-r--r--lib/gitlab/git/repository.rb34
-rw-r--r--lib/gitlab/git/rugged_impl/tree.rb1
-rw-r--r--lib/gitlab/git/tag.rb2
-rw-r--r--lib/gitlab/git/tree.rb4
-rw-r--r--lib/gitlab/git/wraps_gitaly_errors.rb40
-rw-r--r--lib/gitlab/git_access_design.rb21
-rw-r--r--lib/gitlab/git_ref_validator.rb4
-rw-r--r--lib/gitlab/gitaly_client/commit_service.rb16
-rw-r--r--lib/gitlab/gitaly_client/operation_service.rb16
-rw-r--r--lib/gitlab/gitaly_client/ref_service.rb6
-rw-r--r--lib/gitlab/gitaly_client/repository_service.rb8
-rw-r--r--lib/gitlab/github_import/bulk_importing.rb26
-rw-r--r--lib/gitlab/github_import/client.rb4
-rw-r--r--lib/gitlab/github_import/clients/proxy.rb31
-rw-r--r--lib/gitlab/github_import/clients/search_repos.rb45
-rw-r--r--lib/gitlab/github_import/importer/attachments/issues_importer.rb2
-rw-r--r--lib/gitlab/github_import/importer/attachments/merge_requests_importer.rb2
-rw-r--r--lib/gitlab/github_import/importer/attachments/releases_importer.rb2
-rw-r--r--lib/gitlab/github_import/importer/collaborator_importer.rb66
-rw-r--r--lib/gitlab/github_import/importer/collaborators_importer.rb54
-rw-r--r--lib/gitlab/github_import/importer/events/cross_referenced.rb1
-rw-r--r--lib/gitlab/github_import/importer/label_links_importer.rb2
-rw-r--r--lib/gitlab/github_import/importer/labels_importer.rb9
-rw-r--r--lib/gitlab/github_import/importer/milestones_importer.rb10
-rw-r--r--lib/gitlab/github_import/importer/note_attachments_importer.rb26
-rw-r--r--lib/gitlab/github_import/importer/pull_request_merged_by_importer.rb73
-rw-r--r--lib/gitlab/github_import/importer/pull_request_review_importer.rb145
-rw-r--r--lib/gitlab/github_import/importer/pull_requests/all_merged_by_importer.rb59
-rw-r--r--lib/gitlab/github_import/importer/pull_requests/merged_by_importer.rb71
-rw-r--r--lib/gitlab/github_import/importer/pull_requests/review_importer.rb141
-rw-r--r--lib/gitlab/github_import/importer/pull_requests/review_request_importer.rb3
-rw-r--r--lib/gitlab/github_import/importer/pull_requests/review_requests_importer.rb1
-rw-r--r--lib/gitlab/github_import/importer/pull_requests/reviews_importer.rb114
-rw-r--r--lib/gitlab/github_import/importer/pull_requests_merged_by_importer.rb57
-rw-r--r--lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb111
-rw-r--r--lib/gitlab/github_import/importer/releases_importer.rb7
-rw-r--r--lib/gitlab/github_import/importer/repository_importer.rb15
-rw-r--r--lib/gitlab/github_import/job_delay_calculator.rb22
-rw-r--r--lib/gitlab/github_import/markdown/attachment.rb16
-rw-r--r--lib/gitlab/github_import/parallel_scheduling.rb48
-rw-r--r--lib/gitlab/github_import/project_relation_type.rb55
-rw-r--r--lib/gitlab/github_import/representation/collaborator.rb45
-rw-r--r--lib/gitlab/github_import/representation/diff_note.rb2
-rw-r--r--lib/gitlab/github_import/representation/issue.rb3
-rw-r--r--lib/gitlab/github_import/representation/issue_event.rb6
-rw-r--r--lib/gitlab/github_import/representation/lfs_object.rb3
-rw-r--r--lib/gitlab/github_import/representation/note.rb2
-rw-r--r--lib/gitlab/github_import/representation/note_text.rb67
-rw-r--r--lib/gitlab/github_import/representation/pull_request.rb3
-rw-r--r--lib/gitlab/github_import/representation/pull_request_review.rb7
-rw-r--r--lib/gitlab/github_import/representation/pull_requests/review_requests.rb8
-rw-r--r--lib/gitlab/github_import/settings.rb16
-rw-r--r--lib/gitlab/github_import/user_finder.rb20
-rw-r--r--lib/gitlab/gitlab_import/client.rb89
-rw-r--r--lib/gitlab/gitlab_import/importer.rb65
-rw-r--r--lib/gitlab/gitlab_import/project_creator.rb30
-rw-r--r--lib/gitlab/gl_repository.rb5
-rw-r--r--lib/gitlab/gl_repository/repo_type.rb8
-rw-r--r--lib/gitlab/gon_helper.rb10
-rw-r--r--lib/gitlab/grape_logging/loggers/response_logger.rb9
-rw-r--r--lib/gitlab/graphql/authorize/authorize_resource.rb4
-rw-r--r--lib/gitlab/graphql/deprecations/deprecation.rb4
-rw-r--r--lib/gitlab/graphql/loaders/lazy_relation_loader.rb74
-rw-r--r--lib/gitlab/graphql/loaders/lazy_relation_loader/registry.rb75
-rw-r--r--lib/gitlab/graphql/loaders/lazy_relation_loader/relation_proxy.rb52
-rw-r--r--lib/gitlab/graphql/loaders/lazy_relation_loader/top_n_loader.rb83
-rw-r--r--lib/gitlab/graphql/pagination/connections.rb4
-rw-r--r--lib/gitlab/graphql/project/dast_profile_connection_extension.rb9
-rw-r--r--lib/gitlab/graphql/subscriptions/action_cable_with_load_balancing.rb54
-rw-r--r--lib/gitlab/harbor/client.rb6
-rw-r--r--lib/gitlab/hook_data/base_builder.rb30
-rw-r--r--lib/gitlab/http_connection_adapter.rb24
-rw-r--r--lib/gitlab/i18n.rb33
-rw-r--r--lib/gitlab/i18n/pluralization.rb86
-rw-r--r--lib/gitlab/import/errors.rb45
-rw-r--r--lib/gitlab/import/import_failure_service.rb8
-rw-r--r--lib/gitlab/import/metrics.rb28
-rw-r--r--lib/gitlab/import_export/attributes_finder.rb8
-rw-r--r--lib/gitlab/import_export/attributes_permitter.rb2
-rw-r--r--lib/gitlab/import_export/base/relation_factory.rb7
-rw-r--r--lib/gitlab/import_export/base/relation_object_saver.rb31
-rw-r--r--lib/gitlab/import_export/command_line_util.rb14
-rw-r--r--lib/gitlab/import_export/config.rb3
-rw-r--r--lib/gitlab/import_export/file_importer.rb17
-rw-r--r--lib/gitlab/import_export/group/relation_tree_restorer.rb56
-rw-r--r--lib/gitlab/import_export/json/legacy_reader.rb123
-rw-r--r--lib/gitlab/import_export/json/legacy_writer.rb88
-rw-r--r--lib/gitlab/import_export/json/ndjson_reader.rb10
-rw-r--r--lib/gitlab/import_export/json/streaming_serializer.rb30
-rw-r--r--lib/gitlab/import_export/project/import_export.yml79
-rw-r--r--lib/gitlab/import_export/project/object_builder.rb13
-rw-r--r--lib/gitlab/import_export/project/relation_factory.rb28
-rw-r--r--lib/gitlab/import_export/project/relation_tree_restorer.rb10
-rw-r--r--lib/gitlab/import_export/project/sample/relation_tree_restorer.rb2
-rw-r--r--lib/gitlab/import_export/project/tree_restorer.rb21
-rw-r--r--lib/gitlab/import_export/project/tree_saver.rb11
-rw-r--r--lib/gitlab/import_sources.rb5
-rw-r--r--lib/gitlab/incoming_email.rb34
-rw-r--r--lib/gitlab/instrumentation/redis.rb4
-rw-r--r--lib/gitlab/instrumentation/redis_base.rb17
-rw-r--r--lib/gitlab/instrumentation/redis_interceptor.rb10
-rw-r--r--lib/gitlab/instrumentation/zoekt.rb49
-rw-r--r--lib/gitlab/instrumentation_helper.rb24
-rw-r--r--lib/gitlab/issuable/clone/copy_resource_events_service.rb4
-rw-r--r--lib/gitlab/issues/rebalancing/state.rb4
-rw-r--r--lib/gitlab/jira_import/issues_importer.rb4
-rw-r--r--lib/gitlab/jira_import/metadata_collector.rb2
-rw-r--r--lib/gitlab/json.rb2
-rw-r--r--lib/gitlab/json_cache.rb2
-rw-r--r--lib/gitlab/kas/client.rb15
-rw-r--r--lib/gitlab/kas/user_access.rb73
-rw-r--r--lib/gitlab/kubernetes/helm/api.rb126
-rw-r--r--lib/gitlab/kubernetes/helm/pod.rb82
-rw-r--r--lib/gitlab/kubernetes/helm/v2/base_command.rb93
-rw-r--r--lib/gitlab/kubernetes/helm/v2/certificate.rb75
-rw-r--r--lib/gitlab/kubernetes/helm/v2/client_command.rb29
-rw-r--r--lib/gitlab/kubernetes/helm/v2/delete_command.rb38
-rw-r--r--lib/gitlab/kubernetes/helm/v2/init_command.rb45
-rw-r--r--lib/gitlab/kubernetes/helm/v2/install_command.rb87
-rw-r--r--lib/gitlab/kubernetes/helm/v2/patch_command.rb67
-rw-r--r--lib/gitlab/kubernetes/helm/v2/reset_command.rb30
-rw-r--r--lib/gitlab/kubernetes/helm/v3/base_command.rb101
-rw-r--r--lib/gitlab/kubernetes/helm/v3/delete_command.rb35
-rw-r--r--lib/gitlab/kubernetes/helm/v3/install_command.rb80
-rw-r--r--lib/gitlab/kubernetes/helm/v3/patch_command.rb60
-rw-r--r--lib/gitlab/language_detection.rb4
-rw-r--r--lib/gitlab/legacy_github_import/client.rb1
-rw-r--r--lib/gitlab/legacy_github_import/importer.rb47
-rw-r--r--lib/gitlab/legacy_github_import/user_formatter.rb11
-rw-r--r--lib/gitlab/loggable.rb11
-rw-r--r--lib/gitlab/mail_room.rb16
-rw-r--r--lib/gitlab/metrics/dashboard/finder.rb21
-rw-r--r--lib/gitlab/metrics/dashboard/service_selector.rb1
-rw-r--r--lib/gitlab/metrics/requests_rack_middleware.rb2
-rw-r--r--lib/gitlab/metrics/sidekiq_slis.rb39
-rw-r--r--lib/gitlab/metrics/subscribers/action_cable.rb69
-rw-r--r--lib/gitlab/metrics/subscribers/active_record.rb3
-rw-r--r--lib/gitlab/metrics/subscribers/external_http.rb32
-rw-r--r--lib/gitlab/metrics/subscribers/rack_attack.rb5
-rw-r--r--lib/gitlab/metrics/subscribers/rails_cache.rb21
-rw-r--r--lib/gitlab/middleware/compressed_json.rb39
-rw-r--r--lib/gitlab/middleware/go.rb2
-rw-r--r--lib/gitlab/middleware/request_context.rb12
-rw-r--r--lib/gitlab/multi_collection_paginator.rb2
-rw-r--r--lib/gitlab/nav/top_nav_menu_item.rb8
-rw-r--r--lib/gitlab/no_cache_headers.rb1
-rw-r--r--lib/gitlab/observability.rb132
-rw-r--r--lib/gitlab/octokit/middleware.rb7
-rw-r--r--lib/gitlab/omniauth_initializer.rb16
-rw-r--r--lib/gitlab/optimistic_locking.rb5
-rw-r--r--lib/gitlab/other_markup.rb5
-rw-r--r--lib/gitlab/pages/deployment_update.rb11
-rw-r--r--lib/gitlab/pages/random_domain.rb51
-rw-r--r--lib/gitlab/pages/virtual_host_finder.rb71
-rw-r--r--lib/gitlab/pagination/keyset.rb4
-rw-r--r--lib/gitlab/pagination/keyset/paginator.rb2
-rw-r--r--lib/gitlab/patch/database_config.rb18
-rw-r--r--lib/gitlab/patch/draw_route.rb2
-rw-r--r--lib/gitlab/patch/node_loader.rb40
-rw-r--r--lib/gitlab/phabricator_import.rb11
-rw-r--r--lib/gitlab/phabricator_import/cache/map.rb71
-rw-r--r--lib/gitlab/phabricator_import/conduit.rb9
-rw-r--r--lib/gitlab/phabricator_import/conduit/client.rb41
-rw-r--r--lib/gitlab/phabricator_import/conduit/maniphest.rb28
-rw-r--r--lib/gitlab/phabricator_import/conduit/pagination.rb24
-rw-r--r--lib/gitlab/phabricator_import/conduit/response.rb60
-rw-r--r--lib/gitlab/phabricator_import/conduit/tasks_response.rb24
-rw-r--r--lib/gitlab/phabricator_import/conduit/user.rb31
-rw-r--r--lib/gitlab/phabricator_import/conduit/users_response.rb23
-rw-r--r--lib/gitlab/phabricator_import/importer.rb44
-rw-r--r--lib/gitlab/phabricator_import/issues/importer.rb43
-rw-r--r--lib/gitlab/phabricator_import/issues/task_importer.rb61
-rw-r--r--lib/gitlab/phabricator_import/project_creator.rb77
-rw-r--r--lib/gitlab/phabricator_import/representation/task.rb72
-rw-r--r--lib/gitlab/phabricator_import/representation/user.rb25
-rw-r--r--lib/gitlab/phabricator_import/user_finder.rb53
-rw-r--r--lib/gitlab/phabricator_import/worker_state.rb47
-rw-r--r--lib/gitlab/private_commit_email.rb2
-rw-r--r--lib/gitlab/project_authorizations.rb48
-rw-r--r--lib/gitlab/project_template.rb6
-rw-r--r--lib/gitlab/prometheus/internal.rb4
-rw-r--r--lib/gitlab/prometheus/queries/knative_invocation_query.rb42
-rw-r--r--lib/gitlab/puma_logging/json_formatter.rb1
-rw-r--r--lib/gitlab/quick_actions/extractor.rb18
-rw-r--r--lib/gitlab/quick_actions/issue_actions.rb7
-rw-r--r--lib/gitlab/quick_actions/merge_request_actions.rb12
-rw-r--r--lib/gitlab/quick_actions/relate_actions.rb34
-rw-r--r--lib/gitlab/quick_actions/work_item_actions.rb54
-rw-r--r--lib/gitlab/rack_attack.rb2
-rw-r--r--lib/gitlab/rack_attack/instrumented_cache_store.rb33
-rw-r--r--lib/gitlab/rack_attack/store.rb57
-rw-r--r--lib/gitlab/reactive_cache_set_cache.rb8
-rw-r--r--lib/gitlab/redis.rb2
-rw-r--r--lib/gitlab/redis/cache.rb13
-rw-r--r--lib/gitlab/redis/cluster_rate_limiting.rb11
-rw-r--r--lib/gitlab/redis/feature_flag.rb25
-rw-r--r--lib/gitlab/redis/multi_store.rb103
-rw-r--r--lib/gitlab/redis/rate_limiting.rb18
-rw-r--r--lib/gitlab/redis/repository_cache.rb8
-rw-r--r--lib/gitlab/redis/wrapper.rb37
-rw-r--r--lib/gitlab/reference_extractor.rb24
-rw-r--r--lib/gitlab/regex.rb109
-rw-r--r--lib/gitlab/registration_features/password_complexity.rb12
-rw-r--r--lib/gitlab/repository_size_error_message.rb34
-rw-r--r--lib/gitlab/request_context.rb18
-rw-r--r--lib/gitlab/resource_events/assignment_event_recorder.rb61
-rw-r--r--lib/gitlab/runtime.rb2
-rw-r--r--lib/gitlab/saas.rb4
-rw-r--r--lib/gitlab/search_results.rb8
-rw-r--r--lib/gitlab/seeders/ci/runner/runner_fleet_seeder.rb14
-rw-r--r--lib/gitlab/seeders/ci/variables_group_seeder.rb53
-rw-r--r--lib/gitlab/seeders/ci/variables_instance_seeder.rb43
-rw-r--r--lib/gitlab/seeders/ci/variables_project_seeder.rb52
-rw-r--r--lib/gitlab/seeders/project_environment_seeder.rb44
-rw-r--r--lib/gitlab/serializer/ci/variables.rb2
-rw-r--r--lib/gitlab/serverless/service.rb102
-rw-r--r--lib/gitlab/service_desk.rb2
-rw-r--r--lib/gitlab/service_desk_email.rb26
-rw-r--r--lib/gitlab/set_cache.rb8
-rw-r--r--lib/gitlab/setup_helper.rb3
-rw-r--r--lib/gitlab/sidekiq_config.rb7
-rw-r--r--lib/gitlab/sidekiq_config/worker_router.rb5
-rw-r--r--lib/gitlab/sidekiq_daemon/memory_killer.rb293
-rw-r--r--lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb14
-rw-r--r--lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/deduplicates_when_scheduling.rb2
-rw-r--r--lib/gitlab/sidekiq_middleware/server_metrics.rb13
-rw-r--r--lib/gitlab/slash_commands/global_slack_handler.rb70
-rw-r--r--lib/gitlab/slash_commands/incident_management/incident_new.rb6
-rw-r--r--lib/gitlab/slug/environment.rb2
-rw-r--r--lib/gitlab/source.rb43
-rw-r--r--lib/gitlab/spamcheck/client.rb28
-rw-r--r--lib/gitlab/spamcheck/result.rb34
-rw-r--r--lib/gitlab/subscription_portal.rb69
-rw-r--r--lib/gitlab/task_helpers.rb2
-rw-r--r--lib/gitlab/timeless.rb10
-rw-r--r--lib/gitlab/tracking.rb39
-rw-r--r--lib/gitlab/tracking/destinations/database_events_snowplow.rb52
-rw-r--r--lib/gitlab/tracking/destinations/snowplow_micro.rb2
-rw-r--r--lib/gitlab/tracking/standard_context.rb31
-rw-r--r--lib/gitlab/untrusted_regexp.rb40
-rw-r--r--lib/gitlab/url_blocker.rb150
-rw-r--r--lib/gitlab/url_blockers/ip_allowlist_entry.rb23
-rw-r--r--lib/gitlab/usage/metrics/aggregates/aggregate.rb7
-rw-r--r--lib/gitlab/usage/metrics/instrumentations/count_bulk_imports_entities_metric.rb5
-rw-r--r--lib/gitlab/usage/metrics/instrumentations/count_ci_runners_group_type_active_metric.rb15
-rw-r--r--lib/gitlab/usage/metrics/instrumentations/count_ci_runners_group_type_active_online_metric.rb15
-rw-r--r--lib/gitlab/usage/metrics/instrumentations/count_ci_runners_instance_type_active_metric.rb17
-rw-r--r--lib/gitlab/usage/metrics/instrumentations/count_ci_runners_instance_type_active_online_metric.rb15
-rw-r--r--lib/gitlab/usage/metrics/instrumentations/count_ci_runners_metric.rb15
-rw-r--r--lib/gitlab/usage/metrics/instrumentations/count_ci_runners_project_type_active_metric.rb15
-rw-r--r--lib/gitlab/usage/metrics/instrumentations/count_ci_runners_project_type_active_online_metric.rb15
-rw-r--r--lib/gitlab/usage/metrics/instrumentations/database_mode.rb15
-rw-r--r--lib/gitlab/usage/metrics/instrumentations/edition_metric.rb19
-rw-r--r--lib/gitlab/usage/metrics/instrumentations/gitlab_dedicated_metric.rb15
-rw-r--r--lib/gitlab/usage/metrics/instrumentations/incoming_email_encrypted_secrets_enabled_metric.rb2
-rw-r--r--lib/gitlab/usage/metrics/instrumentations/index_inconsistencies_metric.rb46
-rw-r--r--lib/gitlab/usage/metrics/instrumentations/installation_creation_date_approximation_metric.rb15
-rw-r--r--lib/gitlab/usage/metrics/instrumentations/installation_creation_date_metric.rb15
-rw-r--r--lib/gitlab/usage/metrics/instrumentations/installation_type_metric.rb19
-rw-r--r--lib/gitlab/usage/metrics/instrumentations/service_desk_email_encrypted_secrets_enabled_metric.rb2
-rw-r--r--lib/gitlab/usage/metrics/instrumentations/version_metric.rb15
-rw-r--r--lib/gitlab/usage/service_ping_report.rb10
-rw-r--r--lib/gitlab/usage_data.rb84
-rw-r--r--lib/gitlab/usage_data_counters/ci_template_unique_counter.rb4
-rw-r--r--lib/gitlab/usage_data_counters/container_registry_event_counter.rb10
-rw-r--r--lib/gitlab/usage_data_counters/counter_events/package_events.yml1
-rw-r--r--lib/gitlab/usage_data_counters/editor_unique_counter.rb22
-rw-r--r--lib/gitlab/usage_data_counters/gitlab_cli_activity_unique_counter.rb4
-rw-r--r--lib/gitlab/usage_data_counters/hll_redis_counter.rb88
-rw-r--r--lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb18
-rw-r--r--lib/gitlab/usage_data_counters/known_events/analytics.yml26
-rw-r--r--lib/gitlab/usage_data_counters/known_events/ci_templates.yml304
-rw-r--r--lib/gitlab/usage_data_counters/known_events/ci_users.yml6
-rw-r--r--lib/gitlab/usage_data_counters/known_events/code_review_events.yml224
-rw-r--r--lib/gitlab/usage_data_counters/known_events/common.yml150
-rw-r--r--lib/gitlab/usage_data_counters/known_events/container_registry_events.yml11
-rw-r--r--lib/gitlab/usage_data_counters/known_events/ecosystem.yml22
-rw-r--r--lib/gitlab/usage_data_counters/known_events/error_tracking.yml4
-rw-r--r--lib/gitlab/usage_data_counters/known_events/importer_events.yml11
-rw-r--r--lib/gitlab/usage_data_counters/known_events/integrations.yml18
-rw-r--r--lib/gitlab/usage_data_counters/known_events/kubernetes_agent.yml2
-rw-r--r--lib/gitlab/usage_data_counters/known_events/package_events.yml48
-rw-r--r--lib/gitlab/usage_data_counters/known_events/product_analytics.yml4
-rw-r--r--lib/gitlab/usage_data_counters/known_events/quickactions.yml134
-rw-r--r--lib/gitlab/usage_data_counters/known_events/work_items.yml21
-rw-r--r--lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb4
-rw-r--r--lib/gitlab/usage_data_counters/track_unique_events.rb81
-rw-r--r--lib/gitlab/usage_data_counters/work_item_activity_unique_counter.rb2
-rw-r--r--lib/gitlab/usage_data_metrics.rb4
-rw-r--r--lib/gitlab/usage_data_non_sql_metrics.rb7
-rw-r--r--lib/gitlab/usage_data_queries.rb7
-rw-r--r--lib/gitlab/utils/email.rb88
-rw-r--r--lib/gitlab/utils/error_message.rb19
-rw-r--r--lib/gitlab/utils/override.rb4
-rw-r--r--lib/gitlab/utils/strong_memoize.rb21
-rw-r--r--lib/gitlab/utils/uniquify.rb45
-rw-r--r--lib/gitlab/utils/usage_data.rb27
-rw-r--r--lib/gitlab/utils/username_and_email_generator.rb42
-rw-r--r--lib/gitlab/verify/batch_verifier.rb2
-rw-r--r--lib/gitlab/workhorse.rb9
-rw-r--r--lib/gitlab/x509/signature.rb2
-rw-r--r--lib/gitlab_settings.rb17
-rw-r--r--lib/gitlab_settings/options.rb80
-rw-r--r--lib/gitlab_settings/settings.rb40
-rw-r--r--lib/object_storage/config.rb24
-rw-r--r--lib/object_storage/direct_upload.rb9
-rw-r--r--lib/object_storage/pending_direct_upload.rb32
-rw-r--r--lib/peek/views/zoekt.rb47
-rw-r--r--lib/product_analytics/settings.rb38
-rw-r--r--lib/product_analytics/tracker.rb11
-rw-r--r--lib/safe_zip/extract.rb21
-rw-r--r--lib/sidebars/admin/base_menu.rb14
-rw-r--r--lib/sidebars/admin/menus/abuse_reports_menu.rb39
-rw-r--r--lib/sidebars/admin/menus/admin_overview_menu.rb94
-rw-r--r--lib/sidebars/admin/menus/admin_settings_menu.rb151
-rw-r--r--lib/sidebars/admin/menus/analytics_menu.rb53
-rw-r--r--lib/sidebars/admin/menus/applications_menu.rb29
-rw-r--r--lib/sidebars/admin/menus/ci_cd_menu.rb47
-rw-r--r--lib/sidebars/admin/menus/deploy_keys_menu.rb29
-rw-r--r--lib/sidebars/admin/menus/kubernetes_menu.rb34
-rw-r--r--lib/sidebars/admin/menus/labels_menu.rb29
-rw-r--r--lib/sidebars/admin/menus/messages_menu.rb29
-rw-r--r--lib/sidebars/admin/menus/monitoring_menu.rb73
-rw-r--r--lib/sidebars/admin/menus/spam_logs_menu.rb34
-rw-r--r--lib/sidebars/admin/menus/system_hooks_menu.rb29
-rw-r--r--lib/sidebars/admin/panel.rb49
-rw-r--r--lib/sidebars/concerns/render_if_logged_in.rb11
-rw-r--r--lib/sidebars/concerns/super_sidebar_panel.rb53
-rw-r--r--lib/sidebars/context.rb5
-rw-r--r--lib/sidebars/explore/menus/groups_menu.rb34
-rw-r--r--lib/sidebars/explore/menus/projects_menu.rb34
-rw-r--r--lib/sidebars/explore/menus/snippets_menu.rb34
-rw-r--r--lib/sidebars/explore/menus/topics_menu.rb34
-rw-r--r--lib/sidebars/explore/panel.rb39
-rw-r--r--lib/sidebars/groups/menus/ci_cd_menu.rb6
-rw-r--r--lib/sidebars/groups/menus/customer_relations_menu.rb11
-rw-r--r--lib/sidebars/groups/menus/group_information_menu.rb9
-rw-r--r--lib/sidebars/groups/menus/invite_team_members_menu.rb46
-rw-r--r--lib/sidebars/groups/menus/issues_menu.rb22
-rw-r--r--lib/sidebars/groups/menus/kubernetes_menu.rb8
-rw-r--r--lib/sidebars/groups/menus/merge_requests_menu.rb10
-rw-r--r--lib/sidebars/groups/menus/observability_menu.rb18
-rw-r--r--lib/sidebars/groups/menus/packages_registries_menu.rb9
-rw-r--r--lib/sidebars/groups/menus/scope_menu.rb12
-rw-r--r--lib/sidebars/groups/menus/settings_menu.rb10
-rw-r--r--lib/sidebars/groups/panel.rb10
-rw-r--r--lib/sidebars/groups/super_sidebar_menus/analyze_menu.rb33
-rw-r--r--lib/sidebars/groups/super_sidebar_menus/build_menu.rb26
-rw-r--r--lib/sidebars/groups/super_sidebar_menus/code_menu.rb26
-rw-r--r--lib/sidebars/groups/super_sidebar_menus/manage_menu.rb28
-rw-r--r--lib/sidebars/groups/super_sidebar_menus/monitor_menu.rb27
-rw-r--r--lib/sidebars/groups/super_sidebar_menus/operations_menu.rb29
-rw-r--r--lib/sidebars/groups/super_sidebar_menus/plan_menu.rb35
-rw-r--r--lib/sidebars/groups/super_sidebar_menus/secure_menu.rb30
-rw-r--r--lib/sidebars/groups/super_sidebar_panel.rb45
-rw-r--r--lib/sidebars/menu.rb60
-rw-r--r--lib/sidebars/menu_item.rb24
-rw-r--r--lib/sidebars/panel.rb11
-rw-r--r--lib/sidebars/projects/menus/analytics_menu.rb14
-rw-r--r--lib/sidebars/projects/menus/ci_cd_menu.rb18
-rw-r--r--lib/sidebars/projects/menus/confluence_menu.rb8
-rw-r--r--lib/sidebars/projects/menus/deployments_menu.rb11
-rw-r--r--lib/sidebars/projects/menus/external_issue_tracker_menu.rb8
-rw-r--r--lib/sidebars/projects/menus/external_wiki_menu.rb8
-rw-r--r--lib/sidebars/projects/menus/infrastructure_menu.rb50
-rw-r--r--lib/sidebars/projects/menus/invite_team_members_menu.rb47
-rw-r--r--lib/sidebars/projects/menus/issues_menu.rb28
-rw-r--r--lib/sidebars/projects/menus/merge_requests_menu.rb17
-rw-r--r--lib/sidebars/projects/menus/monitor_menu.rb26
-rw-r--r--lib/sidebars/projects/menus/packages_registries_menu.rb28
-rw-r--r--lib/sidebars/projects/menus/project_information_menu.rb8
-rw-r--r--lib/sidebars/projects/menus/repository_menu.rb24
-rw-r--r--lib/sidebars/projects/menus/scope_menu.rb10
-rw-r--r--lib/sidebars/projects/menus/security_compliance_menu.rb10
-rw-r--r--lib/sidebars/projects/menus/settings_menu.rb12
-rw-r--r--lib/sidebars/projects/menus/snippets_menu.rb8
-rw-r--r--lib/sidebars/projects/menus/wiki_menu.rb8
-rw-r--r--lib/sidebars/projects/panel.rb8
-rw-r--r--lib/sidebars/projects/super_sidebar_menus/analyze_menu.rb35
-rw-r--r--lib/sidebars/projects/super_sidebar_menus/build_menu.rb34
-rw-r--r--lib/sidebars/projects/super_sidebar_menus/code_menu.rb34
-rw-r--r--lib/sidebars/projects/super_sidebar_menus/manage_menu.rb28
-rw-r--r--lib/sidebars/projects/super_sidebar_menus/monitor_menu.rb32
-rw-r--r--lib/sidebars/projects/super_sidebar_menus/operations_menu.rb32
-rw-r--r--lib/sidebars/projects/super_sidebar_menus/plan_menu.rb31
-rw-r--r--lib/sidebars/projects/super_sidebar_menus/secure_menu.rb34
-rw-r--r--lib/sidebars/projects/super_sidebar_panel.rb47
-rw-r--r--lib/sidebars/search/panel.rb20
-rw-r--r--lib/sidebars/static_menu.rb13
-rw-r--r--lib/sidebars/uncategorized_menu.rb19
-rw-r--r--lib/sidebars/user_profile/base_menu.rb12
-rw-r--r--lib/sidebars/user_profile/menus/activity_menu.rb29
-rw-r--r--lib/sidebars/user_profile/menus/contributed_projects_menu.rb29
-rw-r--r--lib/sidebars/user_profile/menus/followers_menu.rb43
-rw-r--r--lib/sidebars/user_profile/menus/following_menu.rb43
-rw-r--r--lib/sidebars/user_profile/menus/groups_menu.rb29
-rw-r--r--lib/sidebars/user_profile/menus/overview_menu.rb29
-rw-r--r--lib/sidebars/user_profile/menus/personal_projects_menu.rb29
-rw-r--r--lib/sidebars/user_profile/menus/snippets_menu.rb29
-rw-r--r--lib/sidebars/user_profile/menus/starred_projects_menu.rb29
-rw-r--r--lib/sidebars/user_profile/panel.rb52
-rw-r--r--lib/sidebars/user_settings/menus/access_tokens_menu.rb39
-rw-r--r--lib/sidebars/user_settings/menus/account_menu.rb36
-rw-r--r--lib/sidebars/user_settings/menus/active_sessions_menu.rb31
-rw-r--r--lib/sidebars/user_settings/menus/applications_menu.rb31
-rw-r--r--lib/sidebars/user_settings/menus/authentication_log_menu.rb31
-rw-r--r--lib/sidebars/user_settings/menus/chat_menu.rb31
-rw-r--r--lib/sidebars/user_settings/menus/comment_templates_menu.rb42
-rw-r--r--lib/sidebars/user_settings/menus/emails_menu.rb36
-rw-r--r--lib/sidebars/user_settings/menus/gpg_keys_menu.rb31
-rw-r--r--lib/sidebars/user_settings/menus/notifications_menu.rb31
-rw-r--r--lib/sidebars/user_settings/menus/password_menu.rb39
-rw-r--r--lib/sidebars/user_settings/menus/preferences_menu.rb31
-rw-r--r--lib/sidebars/user_settings/menus/profile_menu.rb31
-rw-r--r--lib/sidebars/user_settings/menus/ssh_keys_menu.rb36
-rw-r--r--lib/sidebars/user_settings/panel.rb51
-rw-r--r--lib/sidebars/your_work/panel.rb9
-rw-r--r--lib/slack/api.rb27
-rw-r--r--lib/slack/block_kit/app_home_opened.rb173
-rw-r--r--lib/slack/block_kit/incident_management/incident_modal_opened.rb343
-rw-r--r--lib/support/nginx/gitlab2
-rw-r--r--lib/support/nginx/gitlab-pages-ssl11
-rw-r--r--lib/support/nginx/gitlab-ssl13
-rw-r--r--lib/support/nginx/registry-ssl11
-rw-r--r--lib/system_check/app/redis_version_check.rb4
-rw-r--r--lib/tasks/benchmark.rake8
-rw-r--r--lib/tasks/contracts/merge_requests.rake2
-rw-r--r--lib/tasks/contracts/pipeline_schedules.rake2
-rw-r--r--lib/tasks/contracts/pipelines.rake2
-rw-r--r--lib/tasks/db_obsolete_ignored_columns.rake4
-rw-r--r--lib/tasks/gettext.rake74
-rw-r--r--lib/tasks/gitlab/assets.rake19
-rw-r--r--lib/tasks/gitlab/background_migrations.rake20
-rw-r--r--lib/tasks/gitlab/backup.rake226
-rw-r--r--lib/tasks/gitlab/bulk_add_permission.rake16
-rw-r--r--lib/tasks/gitlab/db.rake121
-rw-r--r--lib/tasks/gitlab/db/decomposition/connection_status.rake37
-rw-r--r--lib/tasks/gitlab/docs/redirect.rake7
-rw-r--r--lib/tasks/gitlab/feature_categories.rake15
-rw-r--r--lib/tasks/gitlab/gitaly.rake13
-rw-r--r--lib/tasks/gitlab/graphql.rake18
-rw-r--r--lib/tasks/gitlab/import.rake24
-rw-r--r--lib/tasks/gitlab/import_export/import.rake10
-rw-r--r--lib/tasks/gitlab/lfs/migrate.rake4
-rw-r--r--lib/tasks/gitlab/metrics_exporter.rake5
-rw-r--r--lib/tasks/gitlab/openapi.rake4
-rw-r--r--lib/tasks/gitlab/packages/events.rake12
-rw-r--r--lib/tasks/gitlab/packages/migrate.rake4
-rw-r--r--lib/tasks/gitlab/pages.rake4
-rw-r--r--lib/tasks/gitlab/refresh_project_statistics_build_artifacts_size.rake20
-rw-r--r--lib/tasks/gitlab/seed/ci_variables_group.rake28
-rw-r--r--lib/tasks/gitlab/seed/ci_variables_instance.rake23
-rw-r--r--lib/tasks/gitlab/seed/ci_variables_project.rake28
-rw-r--r--lib/tasks/gitlab/seed/project_environments.rake24
-rw-r--r--lib/tasks/gitlab/seed/runner_fleet.rake10
-rw-r--r--lib/tasks/gitlab/setup.rake1
-rw-r--r--lib/tasks/gitlab/terraform/migrate.rake4
-rw-r--r--lib/tasks/gitlab/tw/codeowners.rake57
-rw-r--r--lib/tasks/gitlab/usage_data.rake2
-rw-r--r--lib/tasks/gitlab/x509/update.rake4
-rw-r--r--lib/tasks/import.rake6
-rw-r--r--lib/tasks/tokens.rake10
-rw-r--r--lib/uploaded_file.rb38
1088 files changed, 18311 insertions, 73355 deletions
diff --git a/lib/api/access_requests.rb b/lib/api/access_requests.rb
index 38a9856ca58..249301f308c 100644
--- a/lib/api/access_requests.rb
+++ b/lib/api/access_requests.rb
@@ -8,7 +8,7 @@ module API
helpers ::API::Helpers::MembersHelpers
- feature_category :authentication_and_authorization
+ feature_category :system_access
%w[group project].each do |source_type|
params do
diff --git a/lib/api/admin/ci/variables.rb b/lib/api/admin/ci/variables.rb
index bc351e27f99..3277acc1b52 100644
--- a/lib/api/admin/ci/variables.rb
+++ b/lib/api/admin/ci/variables.rb
@@ -8,7 +8,7 @@ module API
before { authenticated_as_admin! }
- feature_category :pipeline_authoring
+ feature_category :secrets_management
namespace 'admin' do
namespace 'ci' do
diff --git a/lib/api/admin/instance_clusters.rb b/lib/api/admin/instance_clusters.rb
index f848103d9a0..bca991542ed 100644
--- a/lib/api/admin/instance_clusters.rb
+++ b/lib/api/admin/instance_clusters.rb
@@ -5,7 +5,7 @@ module API
class InstanceClusters < ::API::Base
include PaginationParams
- feature_category :kubernetes_management
+ feature_category :deployment_management
urgency :low
before do
diff --git a/lib/api/admin/plan_limits.rb b/lib/api/admin/plan_limits.rb
index 5ef56d3326f..f1d7b56ad92 100644
--- a/lib/api/admin/plan_limits.rb
+++ b/lib/api/admin/plan_limits.rb
@@ -53,7 +53,6 @@ module API
optional :ci_pipeline_size, type: Integer, desc: 'Maximum number of jobs in a single pipeline'
optional :ci_active_jobs, type: Integer, desc: 'Total number of jobs in currently active pipelines'
- optional :ci_active_pipelines, type: Integer, desc: 'Maximum number of active pipelines per project'
optional :ci_project_subscriptions, type: Integer,
desc: 'Maximum number of pipeline subscriptions to and from a project'
optional :ci_pipeline_schedules, type: Integer, desc: 'Maximum number of pipeline schedules'
diff --git a/lib/api/api.rb b/lib/api/api.rb
index 4b7fe6bdc7a..f50c705c3ea 100644
--- a/lib/api/api.rb
+++ b/lib/api/api.rb
@@ -134,6 +134,10 @@ module API
rack_response({ 'message' => '403 Forbidden' }.to_json, 403)
end
+ rescue_from Gitlab::Git::ResourceExhaustedError do |exception|
+ rack_response({ 'message' => exception.message }.to_json, 429, exception.headers)
+ end
+
rescue_from :all do |exception|
handle_api_exception(exception)
end
@@ -234,6 +238,9 @@ module API
mount ::API::ImportBitbucketServer
mount ::API::ImportGithub
mount ::API::Integrations
+ mount ::API::Integrations::Slack::Events
+ mount ::API::Integrations::Slack::Interactions
+ mount ::API::Integrations::Slack::Options
mount ::API::Integrations::JiraConnect::Subscriptions
mount ::API::Invitations
mount ::API::IssueLinks
@@ -265,6 +272,7 @@ module API
mount ::API::ProjectExport
mount ::API::ProjectHooks
mount ::API::ProjectImport
+ mount ::API::ProjectJobTokenScope
mount ::API::ProjectPackages
mount ::API::ProjectRepositoryStorageMoves
mount ::API::ProjectSnippets
@@ -335,7 +343,7 @@ module API
mount ::API::Todos
mount ::API::UsageData
mount ::API::UsageDataNonSqlMetrics
- mount ::API::Ml::Mlflow
+ mount ::API::Ml::Mlflow::Entrypoint
end
mount ::API::Internal::Base
diff --git a/lib/api/applications.rb b/lib/api/applications.rb
index 6fc9408a570..39f1638301b 100644
--- a/lib/api/applications.rb
+++ b/lib/api/applications.rb
@@ -5,7 +5,7 @@ module API
class Applications < ::API::Base
before { authenticated_as_admin! }
- feature_category :authentication_and_authorization
+ feature_category :system_access
resource :applications do
desc 'Create a new application' do
diff --git a/lib/api/branches.rb b/lib/api/branches.rb
index 5ae1a80a7fd..c5ea3a2d3ad 100644
--- a/lib/api/branches.rb
+++ b/lib/api/branches.rb
@@ -14,7 +14,7 @@ module API
before do
require_repository_enabled!
- authorize! :read_code, user_project
+ authorize_read_code!
end
rescue_from Gitlab::Git::Repository::NoRepository do
diff --git a/lib/api/bulk_imports.rb b/lib/api/bulk_imports.rb
index e3dc9ea52cb..b4ace6cd6bc 100644
--- a/lib/api/bulk_imports.rb
+++ b/lib/api/bulk_imports.rb
@@ -59,8 +59,8 @@ module API
requires :entities, type: Array, desc: 'List of entities to import' do
requires :source_type,
type: String,
- desc: 'Source entity type (only `group_entity` is supported)',
- values: %w[group_entity]
+ desc: 'Source entity type',
+ values: %w[group_entity project_entity]
requires :source_full_path,
type: String,
desc: 'Relative path of the source entity to import',
diff --git a/lib/api/ci/helpers/runner.rb b/lib/api/ci/helpers/runner.rb
index 96f5265ce23..7ca8b2df3dd 100644
--- a/lib/api/ci/helpers/runner.rb
+++ b/lib/api/ci/helpers/runner.rb
@@ -12,13 +12,15 @@ module API
JOB_TOKEN_PARAM = :token
LEGACY_SYSTEM_XID = '<legacy>'
- def authenticate_runner!
+ def authenticate_runner!(ensure_runner_manager: true, update_contacted_at: true)
track_runner_authentication
forbidden! unless current_runner
runner_details = get_runner_details_from_request
- current_runner.heartbeat(runner_details)
- current_runner_machine&.heartbeat(runner_details)
+ current_runner.heartbeat(runner_details, update_contacted_at: update_contacted_at)
+ return unless ensure_runner_manager
+
+ current_runner_manager&.heartbeat(runner_details, update_contacted_at: update_contacted_at)
end
def get_runner_details_from_request
@@ -52,12 +54,10 @@ module API
end
end
- def current_runner_machine
- return if Feature.disabled?(:create_runner_machine)
-
- strong_memoize(:current_runner_machine) do
+ def current_runner_manager
+ strong_memoize(:current_runner_manager) do
system_xid = params.fetch(:system_id, LEGACY_SYSTEM_XID)
- current_runner&.ensure_machine(system_xid) { |m| m.contacted_at = Time.current }
+ current_runner&.ensure_manager(system_xid) { |m| m.contacted_at = Time.current }
end
end
@@ -96,7 +96,7 @@ module API
# the heartbeat should be triggered.
if heartbeat_runner
job.runner&.heartbeat(get_runner_ip)
- job.runner_machine&.heartbeat(get_runner_ip)
+ job.runner_manager&.heartbeat(get_runner_ip)
end
job
diff --git a/lib/api/ci/jobs.rb b/lib/api/ci/jobs.rb
index 30d12864bf8..5d60c004a03 100644
--- a/lib/api/ci/jobs.rb
+++ b/lib/api/ci/jobs.rb
@@ -49,7 +49,7 @@ module API
end
# rubocop: disable CodeReuse/ActiveRecord
get ':id/jobs', urgency: :low, feature_category: :continuous_integration do
- check_rate_limit!(:jobs_index, scope: current_user) if enforce_jobs_api_rate_limits(@project)
+ check_rate_limit!(:jobs_index, scope: current_user)
authorize_read_builds!
@@ -250,7 +250,7 @@ module API
]
end
route_setting :authentication, job_token_allowed: true
- get '/allowed_agents', urgency: :low, feature_category: :kubernetes_management do
+ get '/allowed_agents', urgency: :low, feature_category: :deployment_management do
validate_current_authenticated_job
status 200
@@ -266,14 +266,14 @@ module API
persisted_environment = current_authenticated_job.actual_persisted_environment
environment = { tier: persisted_environment.tier, slug: persisted_environment.slug } if persisted_environment
- agent_authorizations = ::Clusters::Agents::FilterAuthorizationsService.new(
- ::Clusters::AgentAuthorizationsFinder.new(project).execute,
+ agent_authorizations = ::Clusters::Agents::Authorizations::CiAccess::FilterService.new(
+ ::Clusters::Agents::Authorizations::CiAccess::Finder.new(project).execute,
environment: persisted_environment&.name
).execute
# See https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/blob/master/doc/kubernetes_ci_access.md#apiv4joballowed_agents-api
{
- allowed_agents: Entities::Clusters::AgentAuthorization.represent(agent_authorizations),
+ allowed_agents: Entities::Clusters::Agents::Authorizations::CiAccess.represent(agent_authorizations),
job: { id: current_authenticated_job.id },
pipeline: { id: pipeline.id },
project: { id: project.id, groups: project_groups },
diff --git a/lib/api/ci/pipeline_schedules.rb b/lib/api/ci/pipeline_schedules.rb
index afb3754f2ae..e27ec24fb44 100644
--- a/lib/api/ci/pipeline_schedules.rb
+++ b/lib/api/ci/pipeline_schedules.rb
@@ -141,9 +141,12 @@ module API
requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id', documentation: { example: 13 }
end
post ':id/pipeline_schedules/:pipeline_schedule_id/take_ownership' do
- authorize! :take_ownership_pipeline_schedule, pipeline_schedule
+ authorize! :admin_pipeline_schedule, pipeline_schedule
- if pipeline_schedule.own!(current_user)
+ if pipeline_schedule.owned_by?(current_user)
+ status(:ok) # Set response code to 200 if schedule is already owned by current user
+ present pipeline_schedule, with: Entities::Ci::PipelineScheduleDetails
+ elsif pipeline_schedule.own!(current_user)
present pipeline_schedule, with: Entities::Ci::PipelineScheduleDetails
else
render_validation_error!(pipeline_schedule)
diff --git a/lib/api/ci/pipelines.rb b/lib/api/ci/pipelines.rb
index c055512e54e..6416de6d2a9 100644
--- a/lib/api/ci/pipelines.rb
+++ b/lib/api/ci/pipelines.rb
@@ -69,13 +69,19 @@ module API
documentation: { example: 'asc' }
optional :source, type: String, values: ::Ci::Pipeline.sources.keys,
documentation: { example: 'push' }
+ optional :name, types: String, desc: 'Filter pipelines by name',
+ documentation: { example: 'Build pipeline' }
end
get ':id/pipelines', urgency: :low, feature_category: :continuous_integration do
authorize! :read_pipeline, user_project
authorize! :read_build, user_project
+ params.delete(:name) unless ::Feature.enabled?(:pipeline_name_in_api, user_project)
+
pipelines = ::Ci::PipelinesFinder.new(user_project, current_user, params).execute
- present paginate(pipelines), with: Entities::Ci::PipelineBasic, project: user_project
+ pipelines = pipelines.preload_pipeline_metadata if ::Feature.enabled?(:pipeline_name_in_api, user_project)
+
+ present paginate(pipelines), with: Entities::Ci::PipelineBasicWithMetadata, project: user_project
end
desc 'Create a new pipeline' do
@@ -119,7 +125,7 @@ module API
desc 'Gets the latest pipeline for the project branch' do
detail 'This feature was introduced in GitLab 12.3'
- success status: 200, model: Entities::Ci::Pipeline
+ success status: 200, model: Entities::Ci::PipelineWithMetadata
failure [
{ code: 401, message: 'Unauthorized' },
{ code: 403, message: 'Forbidden' },
@@ -133,12 +139,12 @@ module API
get ':id/pipelines/latest', urgency: :low, feature_category: :continuous_integration do
authorize! :read_pipeline, latest_pipeline
- present latest_pipeline, with: Entities::Ci::Pipeline
+ present latest_pipeline, with: Entities::Ci::PipelineWithMetadata
end
desc 'Gets a specific pipeline for the project' do
detail 'This feature was introduced in GitLab 8.11'
- success status: 200, model: Entities::Ci::Pipeline
+ success status: 200, model: Entities::Ci::PipelineWithMetadata
failure [
{ code: 401, message: 'Unauthorized' },
{ code: 403, message: 'Forbidden' },
@@ -151,7 +157,7 @@ module API
get ':id/pipelines/:pipeline_id', urgency: :low, feature_category: :continuous_integration do
authorize! :read_pipeline, pipeline
- present pipeline, with: Entities::Ci::Pipeline
+ present pipeline, with: Entities::Ci::PipelineWithMetadata
end
desc 'Get pipeline jobs' do
@@ -199,7 +205,7 @@ module API
use :pagination
end
- get ':id/pipelines/:pipeline_id/bridges', urgency: :low, feature_category: :pipeline_authoring do
+ get ':id/pipelines/:pipeline_id/bridges', urgency: :low, feature_category: :pipeline_composition do
authorize!(:read_build, user_project)
pipeline = user_project.all_pipelines.find(params[:pipeline_id])
@@ -225,7 +231,7 @@ module API
params do
requires :pipeline_id, type: Integer, desc: 'The pipeline ID', documentation: { example: 18 }
end
- get ':id/pipelines/:pipeline_id/variables', feature_category: :pipeline_authoring, urgency: :low do
+ get ':id/pipelines/:pipeline_id/variables', feature_category: :secrets_management, urgency: :low do
authorize! :read_pipeline_variable, pipeline
present pipeline.variables, with: Entities::Ci::Variable
diff --git a/lib/api/ci/runner.rb b/lib/api/ci/runner.rb
index 1c81db39bb1..0247ce301e2 100644
--- a/lib/api/ci/runner.rb
+++ b/lib/api/ci/runner.rb
@@ -11,11 +11,11 @@ module API
desc 'Register a new runner' do
detail "Register a new runner for the instance"
success Entities::Ci::RunnerRegistrationDetails
- failure [[400, 'Bad Request'], [403, 'Forbidden']]
+ failure [[400, 'Bad Request'], [403, 'Forbidden'], [410, 'Gone']]
end
params do
requires :token, type: String, desc: 'Registration token'
- optional :description, type: String, desc: %q(Runner's description)
+ optional :description, type: String, desc: %q(Description of the runner)
optional :maintainer_note, type: String, desc: %q(Deprecated: see `maintenance_note`)
optional :maintenance_note, type: String,
desc: %q(Free-form maintenance notes for the runner (1024 characters))
@@ -27,13 +27,13 @@ module API
optional :architecture, type: String, desc: %q(Runner's architecture)
end
optional :active, type: Boolean,
- desc: 'Deprecated: Use `paused` instead. Specifies whether the runner is allowed ' \
+ desc: 'Deprecated: Use `paused` instead. Specifies if the runner is allowed ' \
'to receive new jobs'
- optional :paused, type: Boolean, desc: 'Specifies whether the runner should ignore new jobs'
- optional :locked, type: Boolean, desc: 'Specifies whether the runner should be locked for the current project'
+ optional :paused, type: Boolean, desc: 'Specifies if the runner should ignore new jobs'
+ optional :locked, type: Boolean, desc: 'Specifies if the runner should be locked for the current project'
optional :access_level, type: String, values: ::Ci::Runner.access_levels.keys,
desc: 'The access level of the runner'
- optional :run_untagged, type: Boolean, desc: 'Specifies whether the runner should handle untagged jobs'
+ optional :run_untagged, type: Boolean, desc: 'Specifies if the runner should handle untagged jobs'
optional :tag_list, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce,
desc: %q(A list of runner tags)
optional :maximum_timeout, type: Integer,
@@ -51,10 +51,18 @@ module API
attributes[:maintenance_note] ||= deprecated_note if deprecated_note
attributes[:active] = !attributes.delete(:paused) if attributes.include?(:paused)
- result = ::Ci::Runners::RegisterRunnerService.new.execute(params[:token], attributes)
- @runner = result.success? ? result.payload[:runner] : nil
- forbidden! unless @runner
+ result = ::Ci::Runners::RegisterRunnerService.new(params[:token], attributes).execute
+ if result.error?
+ case result.reason
+ when :runner_registration_disallowed
+ render_api_error_with_reason!(410, '410 Gone', result.message)
+ else
+ forbidden!(result.message)
+ end
+ end
+
+ @runner = result.payload[:runner]
if @runner.persisted?
present @runner, with: Entities::Ci::RunnerRegistrationDetails
else
@@ -75,6 +83,25 @@ module API
destroy_conditionally!(current_runner) { ::Ci::Runners::UnregisterRunnerService.new(current_runner, params[:token]).execute }
end
+ desc 'Delete a registered runner manager' do
+ summary 'Internal endpoint that deletes a runner manager by authentication token and system ID.'
+ http_codes [[204, 'Runner manager was deleted'], [400, 'Bad Request'], [403, 'Forbidden'], [404, 'Not Found']]
+ end
+ params do
+ requires :token, type: String, desc: %q(The runner's authentication token)
+ requires :system_id, type: String, desc: %q(The runner's system identifier.)
+ end
+ delete '/managers', urgency: :low, feature_category: :runner_fleet do
+ authenticate_runner!(ensure_runner_manager: false)
+
+ destroy_conditionally!(current_runner) do
+ ::Ci::Runners::UnregisterRunnerManagerService.new(
+ current_runner,
+ params[:token],
+ system_id: params[:system_id]).execute
+ end
+ end
+
desc 'Validate authentication credentials' do
summary "Verify authentication for a registered runner"
success Entities::Ci::RunnerRegistrationDetails
@@ -85,7 +112,11 @@ module API
optional :system_id, type: String, desc: %q(The runner's system identifier)
end
post '/verify', urgency: :low, feature_category: :runner do
- authenticate_runner!
+ # For runners that were created in the UI, we want to update the contacted_at value
+ # only when it starts polling for jobs
+ registering_created_runner = params[:token].start_with?(::Ci::Runner::CREATED_RUNNER_TOKEN_PREFIX)
+
+ authenticate_runner!(update_contacted_at: !registering_created_runner)
status 200
present current_runner, with: Entities::Ci::RunnerRegistrationDetails
@@ -137,7 +168,6 @@ module API
optional :certificate, type: String, desc: %q(Session's certificate)
optional :authorization, type: String, desc: %q(Session's authorization)
end
- optional :job_age, type: Integer, desc: %q(Job should be older than passed age in seconds to be ran on runner)
end
# Since we serialize the build output ourselves to ensure Gitaly
@@ -170,7 +200,7 @@ module API
end
new_update = current_runner.ensure_runner_queue_value
- result = ::Ci::RegisterJobService.new(current_runner, current_runner_machine).execute(runner_params)
+ result = ::Ci::RegisterJobService.new(current_runner, current_runner_manager).execute(runner_params)
if result.valid?
if result.build_json
diff --git a/lib/api/ci/runners.rb b/lib/api/ci/runners.rb
index f2f0f32261a..42817c782f4 100644
--- a/lib/api/ci/runners.rb
+++ b/lib/api/ci/runners.rb
@@ -158,11 +158,11 @@ module API
requires :id, type: Integer, desc: 'The ID of a runner'
optional :description, type: String, desc: 'The description of the runner'
optional :active, type: Boolean, desc: 'Deprecated: Use `paused` instead. Flag indicating whether the runner is allowed to receive jobs'
- optional :paused, type: Boolean, desc: 'Specifies whether the runner should ignore new jobs'
+ optional :paused, type: Boolean, desc: 'Specifies if the runner should ignore new jobs'
optional :tag_list, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce,
desc: 'The list of tags for a runner', documentation: { example: "['macos', 'shell']" }
- optional :run_untagged, type: Boolean, desc: 'Specifies whether the runner can execute untagged jobs'
- optional :locked, type: Boolean, desc: 'Specifies whether the runner is locked'
+ optional :run_untagged, type: Boolean, desc: 'Specifies if the runner can execute untagged jobs'
+ optional :locked, type: Boolean, desc: 'Specifies if the runner is locked'
optional :access_level, type: String, values: ::Ci::Runner.access_levels.keys,
desc: 'The access level of the runner'
optional :maximum_timeout, type: Integer,
diff --git a/lib/api/ci/variables.rb b/lib/api/ci/variables.rb
index 5a6b5987228..f5331eb75da 100644
--- a/lib/api/ci/variables.rb
+++ b/lib/api/ci/variables.rb
@@ -8,7 +8,7 @@ module API
before { authenticate! }
before { authorize! :admin_build, user_project }
- feature_category :pipeline_authoring
+ feature_category :secrets_management
helpers ::API::Helpers::VariablesHelpers
diff --git a/lib/api/clusters/agent_tokens.rb b/lib/api/clusters/agent_tokens.rb
index 68eef21903d..9e23756db33 100644
--- a/lib/api/clusters/agent_tokens.rb
+++ b/lib/api/clusters/agent_tokens.rb
@@ -7,7 +7,7 @@ module API
before { authenticate! }
- feature_category :kubernetes_management
+ feature_category :deployment_management
params do
requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project'
@@ -28,7 +28,7 @@ module API
end
get do
agent = ::Clusters::AgentsFinder.new(user_project, current_user).find(params[:agent_id])
- agent_tokens = ::Clusters::AgentTokensFinder.new(agent, current_user).execute
+ agent_tokens = ::Clusters::AgentTokensFinder.new(agent, current_user, status: :active).execute
present paginate(agent_tokens), with: Entities::Clusters::AgentTokenBasic
end
@@ -43,7 +43,7 @@ module API
end
get ':token_id' do
agent = ::Clusters::AgentsFinder.new(user_project, current_user).find(params[:agent_id])
- token = ::Clusters::AgentTokensFinder.new(agent, current_user).find(params[:token_id])
+ token = ::Clusters::AgentTokensFinder.new(agent, current_user, status: :active).find(params[:token_id])
present token, with: Entities::Clusters::AgentToken
end
@@ -65,7 +65,9 @@ module API
agent = ::Clusters::AgentsFinder.new(user_project, current_user).find(params[:agent_id])
result = ::Clusters::AgentTokens::CreateService.new(
- container: agent.project, current_user: current_user, params: token_params.merge(agent_id: agent.id)
+ agent: agent,
+ current_user: current_user,
+ params: token_params
).execute
bad_request!(result[:message]) if result[:status] == :error
@@ -86,8 +88,9 @@ module API
agent = ::Clusters::AgentsFinder.new(user_project, current_user).find(params[:agent_id])
token = ::Clusters::AgentTokensFinder.new(agent, current_user).find(params[:token_id])
- # Skipping explicit error handling and relying on exceptions
- token.revoked!
+ result = ::Clusters::AgentTokens::RevokeService.new(token: token, current_user: current_user).execute
+
+ bad_request!(result[:message]) if result[:status] == :error
status :no_content
end
diff --git a/lib/api/clusters/agents.rb b/lib/api/clusters/agents.rb
index 62d4fb009c6..02469fbad21 100644
--- a/lib/api/clusters/agents.rb
+++ b/lib/api/clusters/agents.rb
@@ -7,7 +7,7 @@ module API
before { authenticate! }
- feature_category :kubernetes_management
+ feature_category :deployment_management
urgency :low
params do
@@ -23,7 +23,7 @@ module API
use :pagination
end
get ':id/cluster_agents' do
- not_found!('ClusterAgents') unless can?(current_user, :read_cluster, user_project)
+ not_found!('ClusterAgents') unless can?(current_user, :read_cluster_agent, user_project)
agents = ::Clusters::AgentsFinder.new(user_project, current_user).execute
diff --git a/lib/api/commits.rb b/lib/api/commits.rb
index fa99a1a2bc8..a4e1e8308c3 100644
--- a/lib/api/commits.rb
+++ b/lib/api/commits.rb
@@ -9,7 +9,7 @@ module API
before do
require_repository_enabled!
- authorize! :read_code, user_project
+ authorize_read_code!
verify_pagination_params!
end
@@ -20,6 +20,8 @@ module API
end
def authorize_push_to_branch!(branch)
+ authenticate!
+
unless user_access.can_push_to_branch?(branch)
forbidden!("You are not allowed to push into this branch")
end
@@ -32,8 +34,6 @@ module API
Gitlab::UsageDataCounters::EditorUniqueCounter.track_web_ide_edit_action(author: current_user, project: user_project)
namespace = user_project.namespace
- return unless Feature.enabled?(:route_hll_to_snowplow_phase3, namespace)
-
Gitlab::Tracking.event(
'API::Commits',
:commit,
@@ -76,6 +76,10 @@ module API
type: String,
desc: 'The file path',
documentation: { example: 'README.md' }
+ optional :author,
+ type: String,
+ desc: 'Search commits by commit author',
+ documentation: { example: 'John Smith' }
optional :all, type: Boolean, desc: 'Every commit will be returned'
optional :with_stats, type: Boolean, desc: 'Stats about each commit will be added to the response'
optional :first_parent, type: Boolean, desc: 'Only include the first parent of merges'
@@ -99,6 +103,7 @@ module API
with_stats = params[:with_stats]
first_parent = params[:first_parent]
order = params[:order]
+ author = params[:author]
commits = user_project.repository.commits(ref,
path: path,
@@ -109,6 +114,7 @@ module API
all: all,
first_parent: first_parent,
order: order,
+ author: author,
trailers: params[:trailers])
serializer = with_stats ? Entities::CommitWithStats : Entities::Commit
@@ -127,6 +133,8 @@ module API
tags %w[commits]
failure [
{ code: 400, message: 'Bad request' },
+ { code: 401, message: 'Unauthorized' },
+ { code: 403, message: 'Forbidden' },
{ code: 404, message: 'Not found' }
]
detail 'This feature was introduced in GitLab 8.13'
diff --git a/lib/api/composer_packages.rb b/lib/api/composer_packages.rb
index 3819e6d236d..56fa10dd7d4 100644
--- a/lib/api/composer_packages.rb
+++ b/lib/api/composer_packages.rb
@@ -179,7 +179,7 @@ module API
.new(authorized_user_project, current_user, declared_params.merge(build: current_authenticated_job))
.execute
- track_package_event('push_package', :composer, project: authorized_user_project, user: current_user, namespace: authorized_user_project.namespace)
+ track_package_event('push_package', :composer, project: authorized_user_project, namespace: authorized_user_project.namespace)
created!
end
diff --git a/lib/api/conan_instance_packages.rb b/lib/api/conan_instance_packages.rb
index 8c13b580092..5f302dba488 100644
--- a/lib/api/conan_instance_packages.rb
+++ b/lib/api/conan_instance_packages.rb
@@ -3,6 +3,12 @@
# Conan Instance-Level Package Manager Client API
module API
class ConanInstancePackages < ::API::Base
+ helpers do
+ def search_project
+ nil
+ end
+ end
+
namespace 'packages/conan/v1' do
include ::API::Concerns::Packages::ConanEndpoints
end
diff --git a/lib/api/conan_project_packages.rb b/lib/api/conan_project_packages.rb
index e282443e85c..a63b3cfd619 100644
--- a/lib/api/conan_project_packages.rb
+++ b/lib/api/conan_project_packages.rb
@@ -3,6 +3,12 @@
# Conan Project-Level Package Manager Client API
module API
class ConanProjectPackages < ::API::Base
+ helpers do
+ def search_project
+ project
+ end
+ end
+
params do
requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project'
end
diff --git a/lib/api/concerns/packages/conan_endpoints.rb b/lib/api/concerns/packages/conan_endpoints.rb
index e65e8f8710c..a7a98482bcc 100644
--- a/lib/api/concerns/packages/conan_endpoints.rb
+++ b/lib/api/concerns/packages/conan_endpoints.rb
@@ -82,7 +82,8 @@ module API
route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true
get 'conans/search', urgency: :low do
- service = ::Packages::Conan::SearchService.new(current_user, query: params[:q]).execute
+ service = ::Packages::Conan::SearchService.new(search_project, current_user, query: params[:q]).execute
+
service.payload
end
@@ -350,7 +351,7 @@ module API
delete urgency: :low do
authorize!(:destroy_package, project)
- track_package_event('delete_package', :conan, category: 'API::ConanPackages', user: current_user, project: project, namespace: project.namespace)
+ track_package_event('delete_package', :conan, category: 'API::ConanPackages', project: project, namespace: project.namespace)
package.destroy
end
diff --git a/lib/api/concerns/packages/debian_package_endpoints.rb b/lib/api/concerns/packages/debian_package_endpoints.rb
index db31f2e35f1..25c97932e31 100644
--- a/lib/api/concerns/packages/debian_package_endpoints.rb
+++ b/lib/api/concerns/packages/debian_package_endpoints.rb
@@ -13,6 +13,7 @@ module API
component: ::Packages::Debian::COMPONENT_REGEX,
architecture: ::Packages::Debian::ARCHITECTURE_REGEX
}.freeze
+ LIST_PACKAGE = 'list_package'
included do
feature_category :package_registry
@@ -40,6 +41,8 @@ module API
package_file = distribution_from!(project).package_files.with_file_name(params[:file_name]).last!
+ track_debian_package_event 'pull_package'
+
present_package_file!(package_file)
end
@@ -58,9 +61,33 @@ module API
.with_compression_type(nil)
.order_created_asc
+ # Empty component files are not persisted in DB
+ no_content! if params[:file_sha256] == ::Packages::Debian::EMPTY_FILE_SHA256
+
relation = relation.with_file_sha256(params[:file_sha256]) if params[:file_sha256]
- present_carrierwave_file!(relation.last!.file)
+ component_file = relation.last
+
+ if component_file.nil? || component_file.empty?
+ not_found! if params[:file_sha256] # asking for a non existing component file.
+ no_content! # empty component files are not always persisted in DB
+ end
+
+ track_debian_package_event LIST_PACKAGE
+
+ present_carrierwave_file!(component_file.file)
+ end
+
+ def track_debian_package_event(action)
+ if project_or_group.is_a?(Project)
+ project = project_or_group
+ namespace = project_or_group.namespace
+ else
+ project = nil
+ namespace = project_or_group
+ end
+
+ track_package_event(action, :debian, project: project, namespace: namespace, user: current_user)
end
end
@@ -73,7 +100,7 @@ module API
end
authenticate_with do |accept|
- accept.token_types(:personal_access_token, :deploy_token, :job_token)
+ accept.token_types(:personal_access_token_with_username, :deploy_token_with_username, :job_token_with_username)
.sent_through(:http_basic_auth)
end
@@ -99,7 +126,6 @@ module API
tags %w[debian_packages]
end
- route_setting :authentication, authenticate_non_public: true
get 'Release.gpg' do
distribution_from!(project_or_group).file_signature
end
@@ -118,9 +144,10 @@ module API
tags %w[debian_packages]
end
- route_setting :authentication, authenticate_non_public: true
get 'Release' do
- present_carrierwave_file!(distribution_from!(project_or_group).file)
+ distribution = distribution_from!(project_or_group)
+ track_debian_package_event LIST_PACKAGE
+ present_carrierwave_file!(distribution.file)
end
# GET {projects|groups}/:id/packages/debian/dists/*distribution/InRelease
@@ -137,9 +164,10 @@ module API
tags %w[debian_packages]
end
- route_setting :authentication, authenticate_non_public: true
get 'InRelease' do
- present_carrierwave_file!(distribution_from!(project_or_group).signed_file)
+ distribution = distribution_from!(project_or_group)
+ track_debian_package_event LIST_PACKAGE
+ present_carrierwave_file!(distribution.signed_file)
end
params do
@@ -156,7 +184,10 @@ module API
# https://wiki.debian.org/DebianRepository/Format#A.22Packages.22_Indices
desc 'The installer (udeb) binary files index' do
detail 'This feature was introduced in GitLab 15.4'
- success code: 200
+ success [
+ { code: 200 },
+ { code: 202 }
+ ]
failure [
{ code: 400, message: 'Bad Request' },
{ code: 401, message: 'Unauthorized' },
@@ -166,7 +197,6 @@ module API
tags %w[debian_packages]
end
- route_setting :authentication, authenticate_non_public: true
get 'Packages' do
present_index_file!(:di_packages)
end
@@ -175,7 +205,10 @@ module API
# https://wiki.debian.org/DebianRepository/Format?action=show&redirect=RepositoryFormat#indices_acquisition_via_hashsums_.28by-hash.29
desc 'The installer (udeb) binary files index by hash' do
detail 'This feature was introduced in GitLab 15.4'
- success code: 200
+ success [
+ { code: 200 },
+ { code: 202 }
+ ]
failure [
{ code: 400, message: 'Bad Request' },
{ code: 401, message: 'Unauthorized' },
@@ -185,7 +218,6 @@ module API
tags %w[debian_packages]
end
- route_setting :authentication, authenticate_non_public: true
get 'by-hash/SHA256/:file_sha256' do
present_index_file!(:di_packages)
end
@@ -196,7 +228,10 @@ module API
# https://wiki.debian.org/DebianRepository/Format#A.22Sources.22_Indices
desc 'The source files index' do
detail 'This feature was introduced in GitLab 15.4'
- success code: 200
+ success [
+ { code: 200 },
+ { code: 202 }
+ ]
failure [
{ code: 400, message: 'Bad Request' },
{ code: 401, message: 'Unauthorized' },
@@ -206,7 +241,6 @@ module API
tags %w[debian_packages]
end
- route_setting :authentication, authenticate_non_public: true
get 'Sources' do
present_index_file!(:sources)
end
@@ -215,7 +249,10 @@ module API
# https://wiki.debian.org/DebianRepository/Format?action=show&redirect=RepositoryFormat#indices_acquisition_via_hashsums_.28by-hash.29
desc 'The source files index by hash' do
detail 'This feature was introduced in GitLab 15.4'
- success code: 200
+ success [
+ { code: 200 },
+ { code: 202 }
+ ]
failure [
{ code: 400, message: 'Bad Request' },
{ code: 401, message: 'Unauthorized' },
@@ -225,7 +262,6 @@ module API
tags %w[debian_packages]
end
- route_setting :authentication, authenticate_non_public: true
get 'by-hash/SHA256/:file_sha256' do
present_index_file!(:sources)
end
@@ -240,7 +276,10 @@ module API
# https://wiki.debian.org/DebianRepository/Format#A.22Packages.22_Indices
desc 'The binary files index' do
detail 'This feature was introduced in GitLab 13.5'
- success code: 200
+ success [
+ { code: 200 },
+ { code: 202 }
+ ]
failure [
{ code: 400, message: 'Bad Request' },
{ code: 401, message: 'Unauthorized' },
@@ -250,7 +289,6 @@ module API
tags %w[debian_packages]
end
- route_setting :authentication, authenticate_non_public: true
get 'Packages' do
present_index_file!(:packages)
end
@@ -259,7 +297,10 @@ module API
# https://wiki.debian.org/DebianRepository/Format?action=show&redirect=RepositoryFormat#indices_acquisition_via_hashsums_.28by-hash.29
desc 'The binary files index by hash' do
detail 'This feature was introduced in GitLab 15.4'
- success code: 200
+ success [
+ { code: 200 },
+ { code: 202 }
+ ]
failure [
{ code: 400, message: 'Bad Request' },
{ code: 401, message: 'Unauthorized' },
@@ -269,7 +310,6 @@ module API
tags %w[debian_packages]
end
- route_setting :authentication, authenticate_non_public: true
get 'by-hash/SHA256/:file_sha256' do
present_index_file!(:packages)
end
diff --git a/lib/api/concerns/packages/npm_endpoints.rb b/lib/api/concerns/packages/npm_endpoints.rb
index f26b3a1d8c2..afbde296161 100644
--- a/lib/api/concerns/packages/npm_endpoints.rb
+++ b/lib/api/concerns/packages/npm_endpoints.rb
@@ -42,6 +42,10 @@ module API
present []
end
end
+
+ def generate_metadata_service(packages)
+ ::Packages::Npm::GenerateMetadataService.new(params[:package_name], packages)
+ end
end
params do
@@ -60,6 +64,7 @@ module API
]
tags %w[npm_packages]
end
+ route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true
get 'dist-tags', format: false, requirements: ::API::Helpers::Packages::Npm::NPM_ENDPOINT_REQUIREMENTS do
package_name = params[:package_name]
@@ -72,7 +77,10 @@ module API
not_found! if packages.empty?
- present ::Packages::Npm::PackagePresenter.new(package_name, packages),
+ track_package_event(:list_tags, :npm, project: project, namespace: project.namespace)
+
+ metadata = generate_metadata_service(packages).execute(only_dist_tags: true)
+ present ::Packages::Npm::PackagePresenter.new(metadata),
with: ::API::Entities::NpmPackageTag
end
@@ -91,6 +99,7 @@ module API
]
tags %w[npm_packages]
end
+ route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true
put format: false do
package_name = params[:package_name]
version = env['api.request.body']
@@ -106,6 +115,8 @@ module API
.find_by_version(version)
not_found!('Package') unless package
+ track_package_event(:create_tag, :npm, project: project, namespace: project.namespace)
+
::Packages::Npm::CreateTagService.new(package, tag).execute
no_content!
@@ -122,6 +133,7 @@ module API
]
tags %w[npm_packages]
end
+ route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true
delete format: false do
package_name = params[:package_name]
tag = params[:tag]
@@ -137,6 +149,8 @@ module API
not_found!('Package tag') unless package_tag
+ track_package_event(:delete_tag, :npm, project: project, namespace: project.namespace)
+
::Packages::RemoveTagService.new(package_tag).execute
no_content!
@@ -163,8 +177,13 @@ module API
route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true
get '*package_name', format: false, requirements: ::API::Helpers::Packages::Npm::NPM_ENDPOINT_REQUIREMENTS do
package_name = params[:package_name]
- packages = ::Packages::Npm::PackageFinder.new(package_name, project: project_or_nil)
- .execute
+ packages =
+ if Feature.enabled?(:npm_allow_packages_in_multiple_projects)
+ finder_for_endpoint_scope(package_name).execute
+ else
+ ::Packages::Npm::PackageFinder.new(package_name, project: project_or_nil)
+ .execute
+ end
redirect_request = project_or_nil.blank? || packages.empty?
@@ -178,7 +197,7 @@ module API
not_found!('Packages') if packages.empty?
- present ::Packages::Npm::PackagePresenter.new(package_name, packages),
+ present ::Packages::Npm::PackagePresenter.new(generate_metadata_service(packages).execute),
with: ::API::Entities::NpmPackage
end
end
diff --git a/lib/api/debian_group_packages.rb b/lib/api/debian_group_packages.rb
index 67416e62f7d..7c64dc2f877 100644
--- a/lib/api/debian_group_packages.rb
+++ b/lib/api/debian_group_packages.rb
@@ -9,16 +9,16 @@ module API
resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
helpers do
def project_or_group
- user_group
+ find_authorized_group!
end
end
after_validation do
require_packages_enabled!
- not_found! unless ::Feature.enabled?(:debian_group_packages, user_group)
+ not_found! unless ::Feature.enabled?(:debian_group_packages, project_or_group)
- authorize_read_package!(user_group)
+ authorize_read_package!(project_or_group)
end
params do
@@ -45,7 +45,6 @@ module API
tags %w[debian_packages]
end
- route_setting :authentication, authenticate_non_public: true
get 'pool/:distribution/:project_id/:letter/:package_name/:package_version/:file_name', requirements: PACKAGE_FILE_REQUIREMENTS do
present_distribution_package_file!(find_project!(params[:project_id]))
end
diff --git a/lib/api/debian_project_packages.rb b/lib/api/debian_project_packages.rb
index 21c0c219046..4f78ac926d8 100644
--- a/lib/api/debian_project_packages.rb
+++ b/lib/api/debian_project_packages.rb
@@ -17,7 +17,7 @@ module API
resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
helpers do
def project_or_group
- user_project(action: :read_package)
+ authorized_user_project(action: :read_package)
end
end
@@ -52,7 +52,6 @@ module API
tags %w[debian_packages]
end
- route_setting :authentication, authenticate_non_public: true
get 'pool/:distribution/:letter/:package_name/:package_version/:file_name', requirements: PACKAGE_FILE_REQUIREMENTS do
present_distribution_package_file!(project_or_group)
end
@@ -82,13 +81,12 @@ module API
optional :distribution, type: String, desc: 'The Debian Codename or Suite', regexp: Gitlab::Regex.debian_distribution_regex
given :distribution do
requires :component, type: String, desc: 'The Debian Component', regexp: Gitlab::Regex.debian_component_regex
- requires :file_name, type: String, desc: 'The filename', regexp: { value: Gitlab::Regex.debian_direct_upload_filename_regex, message: 'Only debs and udebs can be directly added to a distribution' }
+ requires :file_name, type: String, desc: 'The filename', regexp: { value: Gitlab::Regex.debian_direct_upload_filename_regex, message: 'Only debs, udebs and ddebs can be directly added to a distribution' }
end
end
- route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth, authenticate_non_public: true
put do
- authorize_upload!(authorized_user_project)
- bad_request!('File is too large') if authorized_user_project.actual_limits.exceeded?(:debian_max_file_size, params[:file].size)
+ authorize_upload!(project_or_group)
+ bad_request!('File is too large') if project_or_group.actual_limits.exceeded?(:debian_max_file_size, params[:file].size)
file_params = {
file: params['file'],
@@ -101,17 +99,19 @@ module API
package = if params[:distribution].present?
::Packages::CreateTemporaryPackageService.new(
- authorized_user_project, current_user, declared_params.merge(build: current_authenticated_job)
+ project_or_group, current_user, declared_params.merge(build: current_authenticated_job)
).execute(:debian, name: ::Packages::Debian::TEMPORARY_PACKAGE_NAME)
else
- ::Packages::Debian::FindOrCreateIncomingService.new(authorized_user_project, current_user).execute
+ ::Packages::Debian::FindOrCreateIncomingService.new(project_or_group, current_user).execute
end
::Packages::Debian::CreatePackageFileService.new(package: package, current_user: current_user, params: file_params).execute
+ track_debian_package_event 'push_package'
+
created!
rescue ObjectStorage::RemoteStoreError => e
- Gitlab::ErrorTracking.track_exception(e, extra: { file_name: params[:file_name], project_id: authorized_user_project.id })
+ Gitlab::ErrorTracking.track_exception(e, extra: { file_name: params[:file_name], project_id: project_or_group.id })
forbidden!
end
@@ -132,14 +132,13 @@ module API
optional :distribution, type: String, desc: 'The Debian Codename or Suite', regexp: Gitlab::Regex.debian_distribution_regex
given :distribution do
requires :component, type: String, desc: 'The Debian Component', regexp: Gitlab::Regex.debian_component_regex
- requires :file_name, type: String, desc: 'The filename', regexp: { value: Gitlab::Regex.debian_direct_upload_filename_regex, message: 'Only debs and udebs can be directly added to a distribution' }
+ requires :file_name, type: String, desc: 'The filename', regexp: { value: Gitlab::Regex.debian_direct_upload_filename_regex, message: 'Only debs, udebs and ddebs can be directly added to a distribution' }
end
end
- route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth, authenticate_non_public: true
put 'authorize' do
authorize_workhorse!(
- subject: authorized_user_project,
- maximum_size: authorized_user_project.actual_limits.debian_max_file_size
+ subject: project_or_group,
+ maximum_size: project_or_group.actual_limits.debian_max_file_size
)
end
end
diff --git a/lib/api/deploy_keys.rb b/lib/api/deploy_keys.rb
index ffe0b6589bc..634d6052b99 100644
--- a/lib/api/deploy_keys.rb
+++ b/lib/api/deploy_keys.rb
@@ -104,6 +104,7 @@ module API
requires :key, type: String, desc: 'New deploy key'
requires :title, type: String, desc: "New deploy key's title"
optional :can_push, type: Boolean, desc: "Can deploy key push to the project's repository"
+ optional :expires_at, type: DateTime, desc: 'The expiration date of the SSH key in ISO 8601 format (YYYY-MM-DDTHH:MM:SSZ)'
end
# rubocop: disable CodeReuse/ActiveRecord
post ":id/deploy_keys" do
diff --git a/lib/api/draft_notes.rb b/lib/api/draft_notes.rb
index 842180652c4..df9e060e592 100644
--- a/lib/api/draft_notes.rb
+++ b/lib/api/draft_notes.rb
@@ -30,6 +30,28 @@ module API
.new(merge_request(params: params), current_user)
.execute(get_draft_note(params: params))
end
+
+ def publish_draft_notes(params:)
+ ::DraftNotes::PublishService
+ .new(merge_request(params: params), current_user)
+ .execute
+ end
+
+ def authorize_create_note!(params:)
+ access_denied! unless can?(current_user, :create_note, merge_request(params: params))
+ end
+
+ def authorize_admin_draft!(draft_note)
+ access_denied! unless can?(current_user, :admin_note, draft_note)
+ end
+
+ def draft_note_params
+ {
+ note: params[:note],
+ commit_id: params[:commit_id] == 'undefined' ? nil : params[:commit_id],
+ resolve_discussion: params[:resolve_discussion] || false
+ }
+ end
end
resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
@@ -71,6 +93,64 @@ module API
end
end
+ desc "Create a new draft note" do
+ success Entities::DraftNote
+ failure [
+ { code: 401, message: 'Unauthorized' },
+ { code: 404, message: 'Not found' }
+ ]
+ end
+ params do
+ requires :id, type: String, desc: "The ID of a project."
+ requires :merge_request_iid, type: Integer, desc: "The ID of a merge request."
+ requires :note, type: String, desc: 'The content of a note.'
+ optional :in_reply_to_discussion_id, type: Integer, desc: 'The ID of a discussion the draft note replies to.'
+ optional :commit_id, type: String, desc: 'The sha of a commit to associate the draft note to.'
+ optional :resolve_discussion, type: Boolean, desc: 'The associated discussion should be resolved.'
+ end
+ post ":id/merge_requests/:merge_request_iid/draft_notes", feature_category: :code_review_workflow do
+ authorize_create_note!(params: params)
+
+ create_params = draft_note_params.merge(in_reply_to_discussion_id: params[:in_reply_to_discussion_id])
+ create_service = ::DraftNotes::CreateService.new(merge_request(params: params), current_user, create_params)
+
+ draft_note = create_service.execute
+
+ if draft_note.persisted?
+ present draft_note, with: Entities::DraftNote
+ else
+ render_validation_error!(draft_note)
+ end
+ end
+
+ desc "Modify an existing draft note" do
+ success Entities::DraftNote
+ failure [
+ { code: 401, message: 'Unauthorized' },
+ { code: 404, message: 'Not found' }
+ ]
+ end
+ params do
+ requires :id, type: String, desc: "The ID of a project."
+ requires :merge_request_iid, type: Integer, desc: "The ID of a merge request."
+ requires :draft_note_id, type: Integer, desc: "The ID of a draft note"
+ optional :note, type: String, allow_blank: false, desc: 'The content of a note.'
+ end
+ put ":id/merge_requests/:merge_request_iid/draft_notes/:draft_note_id", feature_category: :code_review_workflow do
+ bad_request!('Missing params to modify') unless params[:note].present?
+
+ draft_note = get_draft_note(params: params)
+
+ if draft_note
+ authorize_admin_draft!(draft_note)
+
+ draft_note.update!(note: params[:note])
+ present draft_note, with: Entities::DraftNote
+ else
+ not_found!("Draft Note")
+ end
+ end
+
desc "Delete a draft note" do
success Entities::DraftNote
failure [
@@ -121,6 +201,30 @@ module API
status 500
end
end
+
+ desc "Bulk publish all pending draft notes" do
+ success code: 204
+ failure [
+ { code: 401, message: 'Unauthorized' },
+ { code: 404, message: 'Not found' }
+ ]
+ end
+ params do
+ requires :id, type: String, desc: "The ID of a project"
+ requires :merge_request_iid, type: Integer, desc: "The ID of a merge request"
+ end
+ post(
+ ":id/merge_requests/:merge_request_iid/draft_notes/bulk_publish",
+ feature_category: :code_review_workflow) do
+ result = publish_draft_notes(params: params)
+
+ if result[:status] == :success
+ status 204
+ body false
+ else
+ status 500
+ end
+ end
end
end
end
diff --git a/lib/api/entities/application_setting.rb b/lib/api/entities/application_setting.rb
index 8aace9126d6..91dae5ab825 100644
--- a/lib/api/entities/application_setting.rb
+++ b/lib/api/entities/application_setting.rb
@@ -24,6 +24,7 @@ module API
expose(:default_project_visibility) { |setting, _options| Gitlab::VisibilityLevel.string_level(setting.default_project_visibility) }
expose(:default_snippet_visibility) { |setting, _options| Gitlab::VisibilityLevel.string_level(setting.default_snippet_visibility) }
expose(:default_group_visibility) { |setting, _options| Gitlab::VisibilityLevel.string_level(setting.default_group_visibility) }
+ expose(:valid_runner_registrars) { |setting, _options| setting.valid_runner_registrars }
expose(*::ApplicationSettingsHelper.external_authorization_service_attributes)
diff --git a/lib/api/entities/application_with_secret.rb b/lib/api/entities/application_with_secret.rb
index 1d0acee8624..5679ab4253d 100644
--- a/lib/api/entities/application_with_secret.rb
+++ b/lib/api/entities/application_with_secret.rb
@@ -4,8 +4,12 @@ module API
module Entities
# Use with care, this exposes the secret
class ApplicationWithSecret < Entities::Application
- expose :secret, documentation: { type: 'string',
- example: 'ee1dd64b6adc89cf7e2c23099301ccc2c61b441064e9324d963c46902a85ec34' }
+ expose :secret, documentation: {
+ type: 'string',
+ example: 'ee1dd64b6adc89cf7e2c23099301ccc2c61b441064e9324d963c46902a85ec34'
+ } do |application, _options|
+ application.plaintext_secret
+ end
end
end
end
diff --git a/lib/api/entities/ci/job_request/cache.rb b/lib/api/entities/ci/job_request/cache.rb
index 9820719b4f0..9be2b4c34ce 100644
--- a/lib/api/entities/ci/job_request/cache.rb
+++ b/lib/api/entities/ci/job_request/cache.rb
@@ -5,7 +5,7 @@ module API
module Ci
module JobRequest
class Cache < Grape::Entity
- expose :key, :untracked, :paths, :policy, :when
+ expose :key, :untracked, :paths, :policy, :when, :fallback_keys
end
end
end
diff --git a/lib/api/entities/ci/job_request/response.rb b/lib/api/entities/ci/job_request/response.rb
index cfdbeed79b6..e07bba1e850 100644
--- a/lib/api/entities/ci/job_request/response.rb
+++ b/lib/api/entities/ci/job_request/response.rb
@@ -23,9 +23,7 @@ module API
expose :runner_variables, as: :variables
expose :steps, using: Entities::Ci::JobRequest::Step
- expose :runtime_hooks, as: :hooks,
- using: Entities::Ci::JobRequest::Hook,
- if: ->(job) { ::Feature.enabled?(:ci_hooks_pre_get_sources_script, job.project) }
+ expose :runtime_hooks, as: :hooks, using: Entities::Ci::JobRequest::Hook
expose :image, using: Entities::Ci::JobRequest::Image
expose :services, using: Entities::Ci::JobRequest::Service
expose :artifacts, using: Entities::Ci::JobRequest::Artifacts
diff --git a/lib/api/entities/ci/pipeline_basic_with_metadata.rb b/lib/api/entities/ci/pipeline_basic_with_metadata.rb
new file mode 100644
index 00000000000..4eeba3aec41
--- /dev/null
+++ b/lib/api/entities/ci/pipeline_basic_with_metadata.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module API
+ module Entities
+ module Ci
+ class PipelineBasicWithMetadata < PipelineBasic
+ expose :name,
+ documentation: { type: 'string', example: 'Build pipeline' },
+ if: ->(pipeline, _) { ::Feature.enabled?(:pipeline_name_in_api, pipeline.project) }
+ end
+ end
+ end
+end
diff --git a/lib/api/entities/ci/pipeline_with_metadata.rb b/lib/api/entities/ci/pipeline_with_metadata.rb
new file mode 100644
index 00000000000..a8b1d81a053
--- /dev/null
+++ b/lib/api/entities/ci/pipeline_with_metadata.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module API
+ module Entities
+ module Ci
+ class PipelineWithMetadata < Pipeline
+ expose :name,
+ documentation: { type: 'string', example: 'Build pipeline' },
+ if: ->(pipeline, _) { ::Feature.enabled?(:pipeline_name_in_api, pipeline.project) }
+ end
+ end
+ end
+end
diff --git a/lib/api/entities/ci/secure_file.rb b/lib/api/entities/ci/secure_file.rb
index bf8d2776db1..614fd49aa30 100644
--- a/lib/api/entities/ci/secure_file.rb
+++ b/lib/api/entities/ci/secure_file.rb
@@ -10,7 +10,7 @@ module API
documentation: { type: 'string', example: '16630b189ab34b2e3504f4758e1054d2e478deda510b2b08cc0ef38d12e80aac' }
expose :checksum_algorithm, documentation: { type: 'string', example: 'sha256' }
expose :created_at, documentation: { type: 'dateTime', example: '2022-02-22T22:22:22.222Z' }
- expose :expires_at, documentation: { type: 'dateTime', example: '2022-09-21T14:56:00.000Z' }
+ expose :expires_at, documentation: { type: 'dateTime', example: '2023-09-21T14:55:59.000Z' }
expose :metadata, documentation: { type: 'Hash', example: { "id" => "75949910542696343243264405377658443914" } }
expose :file_extension, documentation: { type: 'string', example: 'jks' }
end
diff --git a/lib/api/entities/clusters/agent_authorization.rb b/lib/api/entities/clusters/agent_authorization.rb
deleted file mode 100644
index 7bbe0f1ec45..00000000000
--- a/lib/api/entities/clusters/agent_authorization.rb
+++ /dev/null
@@ -1,13 +0,0 @@
-# frozen_string_literal: true
-
-module API
- module Entities
- module Clusters
- class AgentAuthorization < Grape::Entity
- expose :agent_id, as: :id
- expose :config_project, with: Entities::ProjectIdentity
- expose :config, as: :configuration
- end
- end
- end
-end
diff --git a/lib/api/entities/clusters/agents/authorizations/ci_access.rb b/lib/api/entities/clusters/agents/authorizations/ci_access.rb
new file mode 100644
index 00000000000..2eefc4361b1
--- /dev/null
+++ b/lib/api/entities/clusters/agents/authorizations/ci_access.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module API
+ module Entities
+ module Clusters
+ module Agents
+ module Authorizations
+ class CiAccess < Grape::Entity
+ expose :agent_id, as: :id
+ expose :config_project, with: Entities::ProjectIdentity
+ expose :config, as: :configuration
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/api/entities/internal/pages/lookup_path.rb b/lib/api/entities/internal/pages/lookup_path.rb
index 1bf94f74fb4..3d16864e587 100644
--- a/lib/api/entities/internal/pages/lookup_path.rb
+++ b/lib/api/entities/internal/pages/lookup_path.rb
@@ -5,8 +5,13 @@ module API
module Internal
module Pages
class LookupPath < Grape::Entity
- expose :project_id, :access_control,
- :source, :https_only, :prefix
+ expose :access_control,
+ :https_only,
+ :prefix,
+ :project_id,
+ :source,
+ :unique_host,
+ :root_directory
end
end
end
diff --git a/lib/api/entities/merge_request_basic.rb b/lib/api/entities/merge_request_basic.rb
index 16afc6c1f6a..f796aeba17f 100644
--- a/lib/api/entities/merge_request_basic.rb
+++ b/lib/api/entities/merge_request_basic.rb
@@ -55,7 +55,10 @@ module API
#
# For list endpoints, we skip the recheck by default, since it's expensive
expose :merge_status do |merge_request, options|
- merge_request.check_mergeability(async: true) unless options[:skip_merge_status_recheck]
+ if !options[:skip_merge_status_recheck] && can_check_mergeability?(merge_request.project)
+ merge_request.check_mergeability(async: true)
+ end
+
merge_request.public_merge_status
end
expose :detailed_merge_status
@@ -101,6 +104,12 @@ module API
def detailed_merge_status
::MergeRequests::Mergeability::DetailedMergeStatusService.new(merge_request: object).execute
end
+
+ def can_check_mergeability?(project)
+ return true if ::Feature.disabled?(:restrict_merge_status_recheck, project)
+
+ Ability.allowed?(options[:current_user], :update_merge_request, project)
+ end
end
end
end
diff --git a/lib/api/entities/ml/mlflow/run_info.rb b/lib/api/entities/ml/mlflow/run_info.rb
index 60e4416e011..6133b3a9d4b 100644
--- a/lib/api/entities/ml/mlflow/run_info.rb
+++ b/lib/api/entities/ml/mlflow/run_info.rb
@@ -19,7 +19,7 @@ module API
private
def run_id
- object.iid.to_s
+ object.eid.to_s
end
end
end
diff --git a/lib/api/entities/note.rb b/lib/api/entities/note.rb
index a92f534bbdc..cac4a8280e3 100644
--- a/lib/api/entities/note.rb
+++ b/lib/api/entities/note.rb
@@ -14,6 +14,7 @@ module API
expose :created_at, :updated_at
expose :system?, as: :system
expose :noteable_id, :noteable_type
+ expose :project_id
expose :commit_id, if: ->(note, options) { note.noteable_type == "MergeRequest" && note.is_a?(DiffNote) }
expose :position, if: ->(note, options) { note.is_a?(DiffNote) } do |note|
diff --git a/lib/api/entities/plan_limit.rb b/lib/api/entities/plan_limit.rb
index d69be0077f2..b5cff2bb73c 100644
--- a/lib/api/entities/plan_limit.rb
+++ b/lib/api/entities/plan_limit.rb
@@ -5,7 +5,6 @@ module API
class PlanLimit < Grape::Entity
expose :ci_pipeline_size, documentation: { type: 'integer', example: 0 }
expose :ci_active_jobs, documentation: { type: 'integer', example: 0 }
- expose :ci_active_pipelines, documentation: { type: 'integer', example: 0 }
expose :ci_project_subscriptions, documentation: { type: 'integer', example: 2 }
expose :ci_pipeline_schedules, documentation: { type: 'integer', example: 10 }
expose :ci_needs_size_limit, documentation: { type: 'integer', example: 50 }
diff --git a/lib/api/entities/project.rb b/lib/api/entities/project.rb
index fcb7ddb9567..61feacd6586 100644
--- a/lib/api/entities/project.rb
+++ b/lib/api/entities/project.rb
@@ -76,7 +76,6 @@ module API
expose(:builds_access_level, documentation: { type: 'string', example: 'enabled' }) { |project, options| project_feature_string_access_level(project, :builds) }
expose(:snippets_access_level, documentation: { type: 'string', example: 'enabled' }) { |project, options| project_feature_string_access_level(project, :snippets) }
expose(:pages_access_level, documentation: { type: 'string', example: 'enabled' }) { |project, options| project_feature_string_access_level(project, :pages) }
- expose(:operations_access_level, documentation: { type: 'string', example: 'enabled' }) { |project, options| project_feature_string_access_level(project, :operations) }
expose(:analytics_access_level, documentation: { type: 'string', example: 'enabled' }) { |project, options| project_feature_string_access_level(project, :analytics) }
expose(:container_registry_access_level, documentation: { type: 'string', example: 'enabled' }) { |project, options| project_feature_string_access_level(project, :container_registry) }
expose(:security_and_compliance_access_level, documentation: { type: 'string', example: 'enabled' }) { |project, options| project_feature_string_access_level(project, :security_and_compliance) }
@@ -88,7 +87,6 @@ module API
expose :emails_disabled, documentation: { type: 'boolean' }
expose :shared_runners_enabled, documentation: { type: 'boolean' }
- expose :group_runners_enabled, documentation: { type: 'boolean' }
expose :lfs_enabled?, as: :lfs_enabled, documentation: { type: 'boolean' }
expose :creator_id, documentation: { type: 'integer', example: 1 }
expose :forked_from_project, using: Entities::BasicProjectDetails, if: ->(project, options) do
@@ -104,30 +102,44 @@ module API
expose :import_error, documentation: { type: 'string', example: 'Import error' }, if: lambda { |_project, options| options[:user_can_admin_project] } do |project|
project.import_state&.last_error
end
-
expose :open_issues_count, documentation: { type: 'integer', example: 1 }, if: lambda { |project, options| project.feature_available?(:issues, options[:current_user]) }
- expose :runners_token, documentation: { type: 'string', example: 'b8547b1dc37721d05889db52fa2f02' }, if: lambda { |_project, options| options[:user_can_admin_project] }
- expose :ci_default_git_depth, documentation: { type: 'integer', example: 20 }
- expose :ci_forward_deployment_enabled, documentation: { type: 'boolean' }
- expose(:ci_job_token_scope_enabled, documentation: { type: 'boolean' }) { |p, _| p.ci_outbound_job_token_scope_enabled? }
- expose :ci_separated_caches, documentation: { type: 'boolean' }
- expose :ci_opt_in_jwt, documentation: { type: 'boolean' }
- expose :ci_allow_fork_pipelines_to_run_in_parent_project, documentation: { type: 'boolean' }
- expose :public_builds, as: :public_jobs, documentation: { type: 'boolean' }
- expose :build_git_strategy, documentation: { type: 'string', example: 'fetch' }, if: lambda { |project, options| options[:user_can_admin_project] } do |project, options|
- project.build_allow_git_fetch ? 'fetch' : 'clone'
+ expose :description_html, documentation: { type: 'string' }
+ expose :updated_at, documentation: { type: 'dateTime', example: '2020-05-07T04:27:17.016Z' }
+
+ with_options if: ->(_, _) { Ability.allowed?(options[:current_user], :admin_project, project) } do
+ # CI/CD Settings
+ expose :ci_default_git_depth, documentation: { type: 'integer', example: 20 }
+ expose :ci_forward_deployment_enabled, documentation: { type: 'boolean' }
+ expose(:ci_job_token_scope_enabled, documentation: { type: 'boolean' }) { |p, _| p.ci_outbound_job_token_scope_enabled? }
+ expose :ci_separated_caches, documentation: { type: 'boolean' }
+ expose :ci_allow_fork_pipelines_to_run_in_parent_project, documentation: { type: 'boolean' }
+ expose :build_git_strategy, documentation: { type: 'string', example: 'fetch' } do |project, options|
+ project.build_allow_git_fetch ? 'fetch' : 'clone'
+ end
+ expose :keep_latest_artifacts_available?, as: :keep_latest_artifact, documentation: { type: 'boolean' }
+ expose :restrict_user_defined_variables, documentation: { type: 'boolean' }
+ expose :runners_token, documentation: { type: 'string', example: 'b8547b1dc37721d05889db52fa2f02' }
+ expose :runner_token_expiration_interval, documentation: { type: 'integer', example: 3600 }
+ expose :group_runners_enabled, documentation: { type: 'boolean' }
+ expose :auto_cancel_pending_pipelines, documentation: { type: 'string', example: 'enabled' }
+ expose :build_timeout, documentation: { type: 'integer', example: 3600 }
+ expose :auto_devops_enabled?, as: :auto_devops_enabled, documentation: { type: 'boolean' }
+ expose :auto_devops_deploy_strategy, documentation: { type: 'string', example: 'continuous' } do |project, options|
+ project.auto_devops.nil? ? 'continuous' : project.auto_devops.deploy_strategy
+ end
end
- expose :build_timeout, documentation: { type: 'integer', example: 3600 }
- expose :auto_cancel_pending_pipelines, documentation: { type: 'string', example: 'enabled' }
+
expose :ci_config_path, documentation: { type: 'string', example: '' }, if: -> (project, options) { Ability.allowed?(options[:current_user], :read_code, project) }
+ expose :public_builds, as: :public_jobs, documentation: { type: 'boolean' }
+
expose :shared_with_groups, documentation: { is_array: true } do |project, options|
user = options[:current_user]
SharedGroupWithProject.represent(project.visible_group_links(for_user: user), options)
end
+
expose :only_allow_merge_if_pipeline_succeeds, documentation: { type: 'boolean' }
expose :allow_merge_on_skipped_pipeline, documentation: { type: 'boolean' }
- expose :restrict_user_defined_variables, documentation: { type: 'boolean' }
expose :request_access_enabled, documentation: { type: 'boolean' }
expose :only_allow_merge_if_all_discussions_are_resolved, documentation: { type: 'boolean' }
expose :remove_source_branch_after_merge, documentation: { type: 'boolean' }
@@ -142,20 +154,15 @@ module API
expose :statistics, using: 'API::Entities::ProjectStatistics', if: -> (project, options) {
options[:statistics] && Ability.allowed?(options[:current_user], :read_statistics, project)
}
- expose :auto_devops_enabled?, as: :auto_devops_enabled, documentation: { type: 'boolean' }
- expose :auto_devops_deploy_strategy, documentation: { type: 'string', example: 'continuous' } do |project, options|
- project.auto_devops.nil? ? 'continuous' : project.auto_devops.deploy_strategy
- end
+
expose :autoclose_referenced_issues, documentation: { type: 'boolean' }
expose :repository_storage, documentation: { type: 'string', example: 'default' }, if: ->(project, options) {
Ability.allowed?(options[:current_user], :change_repository_storage, project)
}
- expose :keep_latest_artifacts_available?, as: :keep_latest_artifact, documentation: { type: 'boolean' }
- expose :runner_token_expiration_interval, documentation: { type: 'integer', example: 3600 }
# rubocop: disable CodeReuse/ActiveRecord
def self.preload_resource(project)
- ActiveRecord::Associations::Preloader.new.preload(project, project_group_links: { group: :route })
+ ActiveRecord::Associations::Preloader.new(records: [project], associations: { project_group_links: { group: :route } }).call
end
def self.preload_relation(projects_relation, options = {})
diff --git a/lib/api/entities/project_job_token_scope.rb b/lib/api/entities/project_job_token_scope.rb
new file mode 100644
index 00000000000..0954db3a2fc
--- /dev/null
+++ b/lib/api/entities/project_job_token_scope.rb
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module API
+ module Entities
+ class ProjectJobTokenScope < Grape::Entity
+ expose(:inbound_enabled, documentation: { type: 'boolean' }) do |project, _|
+ project.ci_inbound_job_token_scope_enabled?
+ end
+ expose(:outbound_enabled, documentation: { type: 'boolean' }) do |project, _|
+ project.ci_outbound_job_token_scope_enabled?
+ end
+ end
+ end
+end
diff --git a/lib/api/entities/protected_ref_access.rb b/lib/api/entities/protected_ref_access.rb
index ba28c724448..28e0ef540d5 100644
--- a/lib/api/entities/protected_ref_access.rb
+++ b/lib/api/entities/protected_ref_access.rb
@@ -9,6 +9,8 @@ module API
documentation: { type: 'string', example: 'Maintainers' } do |protected_ref_access|
protected_ref_access.humanize
end
+ expose :deploy_key_id, documentation: { type: 'integer', example: 1 },
+ if: ->(access) { access.has_attribute?(:deploy_key_id) && access.deploy_key_id }
end
end
end
diff --git a/lib/api/entities/releases/link.rb b/lib/api/entities/releases/link.rb
index 534510ec7e6..0f49440d43b 100644
--- a/lib/api/entities/releases/link.rb
+++ b/lib/api/entities/releases/link.rb
@@ -18,7 +18,6 @@ module API
} do |link|
::Releases::LinkPresenter.new(link).direct_asset_url
end
- expose :external?, documentation: { type: 'boolean' }, as: :external # @deprecated
expose :link_type, documentation: { type: 'string', example: 'other' }
end
end
diff --git a/lib/api/entities/user.rb b/lib/api/entities/user.rb
index 884f0f75d7a..689996be7fc 100644
--- a/lib/api/entities/user.rb
+++ b/lib/api/entities/user.rb
@@ -5,6 +5,7 @@ module API
class User < UserBasic
include UsersHelper
include TimeZoneHelper
+ include Gitlab::Utils::StrongMemoize
expose :created_at, if: ->(user, opts) { Ability.allowed?(opts[:current_user], :read_user_profile, user) }
expose :bio, :location, :public_email, :skype, :linkedin, :twitter, :discord, :website_url, :organization, :job_title, :pronouns
@@ -12,18 +13,28 @@ module API
expose :work_information do |user|
work_information(user)
end
- expose :followers, if: ->(user, opts) { Ability.allowed?(opts[:current_user], :read_user_profile, user) } do |user|
+ expose :followers, if: ->(user, opts) { Ability.allowed?(opts[:current_user], :read_user_profile, user) && following_users_allowed(opts[:current_user], user) } do |user|
user.followers.size
end
- expose :following, if: ->(user, opts) { Ability.allowed?(opts[:current_user], :read_user_profile, user) } do |user|
+ expose :following, if: ->(user, opts) { Ability.allowed?(opts[:current_user], :read_user_profile, user) && following_users_allowed(opts[:current_user], user) } do |user|
user.followees.size
end
- expose :is_followed, if: ->(user, opts) { Ability.allowed?(opts[:current_user], :read_user_profile, user) && opts[:current_user] } do |user, opts|
+ expose :is_followed, if: ->(user, opts) { Ability.allowed?(opts[:current_user], :read_user_profile, user) && opts[:current_user] && following_users_allowed(opts[:current_user], user) } do |user, opts|
user.followed_by?(opts[:current_user])
end
expose :local_time do |user|
local_time(user.timezone)
end
+
+ def following_users_allowed(current_user, user)
+ strong_memoize(:following_users_allowed) do
+ if current_user
+ current_user.following_users_allowed?(user)
+ else
+ true
+ end
+ end
+ end
end
end
end
diff --git a/lib/api/entities/user_preferences.rb b/lib/api/entities/user_preferences.rb
index ceee6c610d3..e66e83549b2 100644
--- a/lib/api/entities/user_preferences.rb
+++ b/lib/api/entities/user_preferences.rb
@@ -3,7 +3,7 @@
module API
module Entities
class UserPreferences < Grape::Entity
- expose :id, :user_id, :view_diffs_file_by_file, :show_whitespace_in_diffs
+ expose :id, :user_id, :view_diffs_file_by_file, :show_whitespace_in_diffs, :pass_user_identities_to_ci_jwt
end
end
end
diff --git a/lib/api/environments.rb b/lib/api/environments.rb
index 64510a9615a..bb261079d2a 100644
--- a/lib/api/environments.rb
+++ b/lib/api/environments.rb
@@ -41,7 +41,7 @@ module API
get ':id/environments' do
authorize! :read_environment, user_project
- if Feature.enabled?(:environment_search_api_min_chars, user_project) && params[:search].present? && params[:search].length < MIN_SEARCH_LENGTH
+ if params[:search].present? && params[:search].length < MIN_SEARCH_LENGTH
bad_request!("Search query is less than #{MIN_SEARCH_LENGTH} characters")
end
@@ -90,8 +90,6 @@ module API
end
params do
requires :environment_id, type: Integer, desc: 'The ID of the environment'
- # TODO: disallow renaming via the API https://gitlab.com/gitlab-org/gitlab/-/issues/338897
- optional :name, type: String, desc: 'DEPRECATED: Renaming environment can lead to errors, this will be removed in 15.0'
optional :external_url, type: String, desc: 'The new URL on which this deployment is viewable'
optional :slug, absence: { message: "is automatically generated and cannot be changed" }, documentation: { hidden: true }
optional :tier, type: String, values: Environment.tiers.keys, desc: 'The tier of the new environment. Allowed values are `production`, `staging`, `testing`, `development`, and `other`'
@@ -101,8 +99,11 @@ module API
environment = user_project.environments.find(params[:environment_id])
- update_params = declared_params(include_missing: false).extract!(:name, :external_url, :tier)
- if environment.update(update_params)
+ update_params = declared_params(include_missing: false).extract!(:external_url, :tier)
+
+ environment.assign_attributes(update_params)
+
+ if environment.save
present environment, with: Entities::Environment, current_user: current_user
else
render_validation_error!(environment)
diff --git a/lib/api/error_tracking/project_settings.rb b/lib/api/error_tracking/project_settings.rb
index ec1d6a8b87f..99af8494723 100644
--- a/lib/api/error_tracking/project_settings.rb
+++ b/lib/api/error_tracking/project_settings.rb
@@ -23,8 +23,6 @@ module API
resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
before do
authorize! :admin_operations, user_project
-
- not_found!('Error Tracking Setting') unless project_setting
end
desc 'Get Error Tracking settings' do
@@ -34,6 +32,7 @@ module API
end
get ':id/error_tracking/settings' do
+ not_found!('Error Tracking Setting') unless project_setting
present project_setting, with: Entities::ErrorTracking::ProjectSetting
end
@@ -59,6 +58,7 @@ module API
end
patch ':id/error_tracking/settings/' do
+ not_found!('Error Tracking Setting') unless project_setting
update_params = {
error_tracking_setting_attributes: { enabled: params[:active] }
}
@@ -75,6 +75,42 @@ module API
result
end
end
+
+ desc 'Update Error Tracking project settings. Available in GitLab 15.10 and later.' do
+ detail 'Update Error Tracking settings for a project. ' \
+ 'Only for users with Maintainer role for the project.'
+ success Entities::ErrorTracking::ProjectSetting
+ failure [
+ { code: 400, message: 'Bad request' },
+ { code: 401, message: 'Unauthorized' },
+ { code: 404, message: 'Not found' }
+ ]
+ tags ERROR_TRACKING_PROJECT_SETTINGS_TAGS
+ end
+ params do
+ requires :active, type: Boolean,
+ desc: 'Pass true to enable the configured Error Tracking settings or false to disable it.',
+ allow_blank: false
+ requires :integrated,
+ type: Boolean,
+ desc: 'Pass true to enable the integrated Error Tracking backend.'
+ end
+
+ put ':id/error_tracking/settings' do
+ not_found! unless Feature.enabled?(:integrated_error_tracking, user_project)
+ update_params = {
+ error_tracking_setting_attributes: { enabled: params[:active] }
+ }
+
+ update_params[:error_tracking_setting_attributes][:integrated] = params[:integrated]
+
+ result = ::Projects::Operations::UpdateService.new(user_project, current_user, update_params).execute
+ if result[:status] == :success
+ present project_setting, with: Entities::ErrorTracking::ProjectSetting
+ else
+ result
+ end
+ end
end
end
end
diff --git a/lib/api/files.rb b/lib/api/files.rb
index 18638abd184..45e935d7ea2 100644
--- a/lib/api/files.rb
+++ b/lib/api/files.rb
@@ -30,7 +30,7 @@ module API
end
def assign_file_vars!
- authorize! :read_code, user_project
+ authorize_read_code!
@commit = user_project.commit(params[:ref])
not_found!('Commit') unless @commit
@@ -49,13 +49,21 @@ module API
end
def content_sha
- Rails.cache.fetch("blob_content_sha256:#{user_project.full_path}:#{@blob.id}") do
+ cache_client.fetch("blob_content_sha256:#{user_project.full_path}:#{@blob.id}") do
@blob.load_all_data!
Digest::SHA256.hexdigest(@blob.data)
end
end
+ def cache_client
+ Gitlab::Cache::Client.build_with_metadata(
+ cache_identifier: 'API::Files#content_sha',
+ feature_category: :source_code_management,
+ backing_resource: :gitaly
+ )
+ end
+
def fetch_blame_range(blame_params)
return if blame_params[:range].blank?
diff --git a/lib/api/generic_packages.rb b/lib/api/generic_packages.rb
index da5b0930543..95852848f3a 100644
--- a/lib/api/generic_packages.rb
+++ b/lib/api/generic_packages.rb
@@ -84,7 +84,7 @@ module API
authorize_upload!(project)
bad_request!('File is too large') if max_file_size_exceeded?
- track_package_event('push_package', :generic, project: project, user: current_user, namespace: project.namespace)
+ track_package_event('push_package', :generic, project: project, namespace: project.namespace)
create_package_file_params = declared_params.merge(build: current_authenticated_job)
package_file = ::Packages::Generic::CreatePackageFileService
@@ -131,7 +131,7 @@ module API
package = ::Packages::Generic::PackageFinder.new(project).execute!(params[:package_name], params[:package_version])
package_file = ::Packages::PackageFileFinder.new(package, params[:file_name]).execute!
- track_package_event('pull_package', :generic, project: project, user: current_user, namespace: project.namespace)
+ track_package_event('pull_package', :generic, project: project, namespace: project.namespace)
present_package_file!(package_file)
end
diff --git a/lib/api/group_clusters.rb b/lib/api/group_clusters.rb
index de5ca0f86ae..9cc7d0b8bf8 100644
--- a/lib/api/group_clusters.rb
+++ b/lib/api/group_clusters.rb
@@ -9,7 +9,7 @@ module API
ensure_feature_enabled!
end
- feature_category :kubernetes_management
+ feature_category :deployment_management
urgency :low
params do
diff --git a/lib/api/group_container_repositories.rb b/lib/api/group_container_repositories.rb
index 753f0db10c1..5173a55308d 100644
--- a/lib/api/group_container_repositories.rb
+++ b/lib/api/group_container_repositories.rb
@@ -38,7 +38,7 @@ module API
user: current_user, subject: user_group
).execute
- track_package_event('list_repositories', :container, user: current_user, namespace: user_group)
+ track_package_event('list_repositories', :container, namespace: user_group)
present paginate(repositories), with: Entities::ContainerRegistry::Repository, tags: false, tags_count: false
end
diff --git a/lib/api/group_variables.rb b/lib/api/group_variables.rb
index a42f9045b9d..295bee475c3 100644
--- a/lib/api/group_variables.rb
+++ b/lib/api/group_variables.rb
@@ -6,7 +6,7 @@ module API
before { authenticate! }
before { authorize! :admin_group, user_group }
- feature_category :pipeline_authoring
+ feature_category :secrets_management
helpers ::API::Helpers::VariablesHelpers
diff --git a/lib/api/groups.rb b/lib/api/groups.rb
index 23db10dbdbf..e13b661b357 100644
--- a/lib/api/groups.rb
+++ b/lib/api/groups.rb
@@ -123,11 +123,7 @@ module API
end
def present_groups_with_pagination_strategies(params, groups)
- # Prevent Rails from optimizing the count query and inadvertadly creating a poor performing databse query.
- # https://gitlab.com/gitlab-org/gitlab/-/issues/368969
- if Feature.enabled?(:present_groups_select_all)
- groups = groups.select(groups.arel_table[Arel.star])
- end
+ groups = groups.select(groups.arel_table[Arel.star])
return present_groups(params, groups) if current_user.present?
diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb
index aadcbe38b15..9fa0923d914 100644
--- a/lib/api/helpers.rb
+++ b/lib/api/helpers.rb
@@ -167,10 +167,6 @@ module API
current_authenticated_job.project == project
end
- def enforce_jobs_api_rate_limits(project)
- ::Feature.enabled?(:ci_enforce_rate_limits_jobs_api, project)
- end
-
# rubocop: disable CodeReuse/ActiveRecord
def find_group(id)
if id.to_s =~ INTEGER_ID_REGEX
@@ -332,6 +328,10 @@ module API
authorize! :read_build, user_project
end
+ def authorize_read_code!
+ authorize! :read_code, user_project
+ end
+
def authorize_read_build_trace!(build)
authorize! :read_build_trace, build
end
@@ -683,6 +683,8 @@ module API
finder_params[:user] = params.delete(:user) if params[:user]
finder_params[:id_after] = sanitize_id_param(params[:id_after]) if params[:id_after]
finder_params[:id_before] = sanitize_id_param(params[:id_before]) if params[:id_before]
+ finder_params[:updated_after] = declared_params[:updated_after] if declared_params[:updated_after]
+ finder_params[:updated_before] = declared_params[:updated_before] if declared_params[:updated_before]
finder_params
end
diff --git a/lib/api/helpers/integrations_helpers.rb b/lib/api/helpers/integrations_helpers.rb
index 31328facd69..4c37a2a5aba 100644
--- a/lib/api/helpers/integrations_helpers.rb
+++ b/lib/api/helpers/integrations_helpers.rb
@@ -191,6 +191,12 @@ module API
name: :app_store_private_key,
type: String,
desc: 'The Apple App Store Connect Private Key'
+ },
+ {
+ required: true,
+ name: :app_store_private_key_file_name,
+ type: String,
+ desc: 'The Apple App Store Connect Private Key File Name'
}
],
'asana' => [
@@ -447,6 +453,26 @@ module API
desc: 'The URL of the external wiki'
}
],
+ 'google-play' => [
+ {
+ required: true,
+ name: :package_name,
+ type: String,
+ desc: 'The package name of the app in Google Play'
+ },
+ {
+ required: true,
+ name: :service_account_key,
+ type: String,
+ desc: 'The Google Play service account key'
+ },
+ {
+ required: true,
+ name: :service_account_key_file_name,
+ type: String,
+ desc: 'The filename of the Google Play service account key'
+ }
+ ],
'hangouts-chat' => [
{
required: true,
@@ -566,16 +592,22 @@ module API
desc: 'The base URL to the Jira instance API. Web URL value will be used if not set. E.g., https://jira-api.example.com'
},
{
- required: true,
+ required: false,
+ name: :jira_auth_type,
+ type: Integer,
+ desc: 'The authentication method to be used with Jira. `0` means Basic Authentication. `1` means Jira personal access token. Defaults to `0`'
+ },
+ {
+ required: false,
name: :username,
type: String,
- desc: 'The username of the user created to be used with GitLab/Jira'
+ desc: 'The email or username to be used with Jira. For Jira Cloud use an email, for Jira Data Center and Jira Server use a username. Required when using Basic authentication (`jira_auth_type` is `0`)'
},
{
required: true,
name: :password,
type: String,
- desc: 'The password of the user created to be used with GitLab/Jira'
+ desc: 'The Jira API token, password, or personal access token to be used with Jira. When your authentication method is Basic (`jira_auth_type` is `0`) use an API token for Jira Cloud, or a password for Jira Data Center or Jira Server. When your authentication method is Jira personal access token (`jira_auth_type` is `1`) use a personal access token'
},
{
required: false,
@@ -591,6 +623,18 @@ module API
},
{
required: false,
+ name: :jira_issue_prefix,
+ type: String,
+ desc: 'Prefix to match Jira issue keys'
+ },
+ {
+ required: false,
+ name: :jira_issue_regex,
+ type: String,
+ desc: 'Regular expression to match Jira issue keys'
+ },
+ {
+ required: false,
name: :comment_on_event_enabled,
type: Boolean,
desc: 'Enable comments inside Jira issues on each GitLab event (commit / merge request)'
@@ -897,6 +941,20 @@ module API
type: String,
desc: 'The product ID of ZenTao project'
}
+ ],
+ 'squash-tm' => [
+ {
+ required: true,
+ name: :url,
+ type: String,
+ desc: 'The Squash TM webhook URL'
+ },
+ {
+ required: false,
+ name: :token,
+ type: String,
+ desc: 'The secret token'
+ }
]
}
end
@@ -918,6 +976,7 @@ module API
::Integrations::EmailsOnPush,
::Integrations::Ewm,
::Integrations::ExternalWiki,
+ ::Integrations::GooglePlay,
::Integrations::HangoutsChat,
::Integrations::Harbor,
::Integrations::Irker,
@@ -934,6 +993,7 @@ module API
::Integrations::Redmine,
::Integrations::Slack,
::Integrations::SlackSlashCommands,
+ ::Integrations::SquashTm,
::Integrations::Teamcity,
::Integrations::Youtrack,
::Integrations::Zentao
diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb
index 816b8deb461..0a6b288e3f8 100644
--- a/lib/api/helpers/internal_helpers.rb
+++ b/lib/api/helpers/internal_helpers.rb
@@ -57,6 +57,7 @@ module API
def log_user_activity(actor)
commands = Gitlab::GitAccess::DOWNLOAD_COMMANDS
+ commands += Gitlab::GitAccess::PUSH_COMMANDS if Feature.enabled?(:log_user_git_push_activity)
return unless commands.include?(params[:action])
diff --git a/lib/api/helpers/members_helpers.rb b/lib/api/helpers/members_helpers.rb
index 4b34a2bbe79..b3c79486465 100644
--- a/lib/api/helpers/members_helpers.rb
+++ b/lib/api/helpers/members_helpers.rb
@@ -43,8 +43,7 @@ module API
end
def source_members(source)
- return source.namespace_members if source.is_a?(Project) &&
- Feature.enabled?(:project_members_index_by_project_namespace, source)
+ return source.namespace_members if source.is_a?(Project)
source.members
end
@@ -56,7 +55,8 @@ module API
end
def find_all_members_for_project(project)
- MembersFinder.new(project, current_user).execute(include_relations: [:inherited, :direct, :invited_groups])
+ include_relations = [:inherited, :direct, :invited_groups, :shared_into_ancestors]
+ MembersFinder.new(project, current_user).execute(include_relations: include_relations)
end
def find_all_members_for_group(group)
diff --git a/lib/api/helpers/merge_requests_helpers.rb b/lib/api/helpers/merge_requests_helpers.rb
index ee3bb49c97f..0a0d70520ef 100644
--- a/lib/api/helpers/merge_requests_helpers.rb
+++ b/lib/api/helpers/merge_requests_helpers.rb
@@ -105,6 +105,9 @@ module API
documentation: { example: '2019-03-15T08:00:00Z' }
optional :environment, desc: 'Returns merge requests deployed to the given environment',
documentation: { example: '2019-03-15T08:00:00Z' }
+ optional :approved, type: String,
+ values: %w[yes no],
+ desc: 'Filters merge requests by their `approved` status. `yes` returns only approved merge requests. `no` returns only non-approved merge requests.'
end
params :optional_scope_param do
diff --git a/lib/api/helpers/packages/conan/api_helpers.rb b/lib/api/helpers/packages/conan/api_helpers.rb
index 3ea558f3569..b47bfbfb5aa 100644
--- a/lib/api/helpers/packages/conan/api_helpers.rb
+++ b/lib/api/helpers/packages/conan/api_helpers.rb
@@ -171,7 +171,7 @@ module API
conan_package_reference: params[:conan_package_reference]
).execute!
- track_package_event('pull_package', :conan, category: 'API::ConanPackages', user: current_user, project: project, namespace: project.namespace) if params[:file_name] == ::Packages::Conan::FileMetadatum::PACKAGE_BINARY
+ track_package_event('pull_package', :conan, category: 'API::ConanPackages', project: project, namespace: project.namespace) if params[:file_name] == ::Packages::Conan::FileMetadatum::PACKAGE_BINARY
present_package_file!(package_file)
end
@@ -186,7 +186,7 @@ module API
def track_push_package_event
if params[:file_name] == ::Packages::Conan::FileMetadatum::PACKAGE_BINARY && params[:file].size > 0 # rubocop: disable Style/ZeroLengthPredicate
- track_package_event('push_package', :conan, category: 'API::ConanPackages', user: current_user, project: project, namespace: project.namespace)
+ track_package_event('push_package', :conan, category: 'API::ConanPackages', project: project, namespace: project.namespace)
end
end
diff --git a/lib/api/helpers/packages/dependency_proxy_helpers.rb b/lib/api/helpers/packages/dependency_proxy_helpers.rb
index 4b0e63c8f3b..a8e5560e106 100644
--- a/lib/api/helpers/packages/dependency_proxy_helpers.rb
+++ b/lib/api/helpers/packages/dependency_proxy_helpers.rb
@@ -28,7 +28,7 @@ module API
end
def registry_url(package_type, options)
- base_url = REGISTRY_BASE_URLS[package_type]
+ base_url = registry_base_url(package_type)
raise ArgumentError, "Can't build registry_url for package_type #{package_type}" unless base_url
@@ -66,7 +66,14 @@ module API
Feature.enabled?(:maven_central_request_forwarding, target.root_ancestor)
end
+
+ # Override in JiHu repo
+ def registry_base_url(package_type)
+ REGISTRY_BASE_URLS[package_type]
+ end
end
end
end
end
+
+API::Helpers::Packages::DependencyProxyHelpers.prepend_mod
diff --git a/lib/api/helpers/packages/maven/basic_auth_helpers.rb b/lib/api/helpers/packages/maven/basic_auth_helpers.rb
new file mode 100644
index 00000000000..c9ef95adc33
--- /dev/null
+++ b/lib/api/helpers/packages/maven/basic_auth_helpers.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module API
+ module Helpers
+ module Packages
+ module Maven
+ module BasicAuthHelpers
+ include ::API::Helpers::Packages::BasicAuthHelpers
+ extend ::Gitlab::Utils::Override
+
+ # override so that we can receive the job token either by headers or
+ # basic auth.
+ override :find_user_from_job_token
+ def find_user_from_job_token
+ super || find_user_from_basic_auth_job
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/api/helpers/packages/npm.rb b/lib/api/helpers/packages/npm.rb
index 352d77f472c..4eb6c39b7dc 100644
--- a/lib/api/helpers/packages/npm.rb
+++ b/lib/api/helpers/packages/npm.rb
@@ -33,6 +33,15 @@ module API
end
end
+ def finder_for_endpoint_scope(package_name)
+ case endpoint_scope
+ when :project
+ ::Packages::Npm::PackageFinder.new(package_name, project: project_or_nil)
+ when :instance
+ ::Packages::Npm::PackageFinder.new(package_name, namespace: top_namespace_from(package_name))
+ end
+ end
+
def project_or_nil
# mainly used by the metadata endpoint where we need to get a project
# and return nil if not found (no errors should be raised)
@@ -50,11 +59,17 @@ module API
params[:id]
when :instance
package_name = params[:package_name]
- namespace_path = ::Packages::Npm.scope_of(package_name)
- next unless namespace_path
- namespace = Namespace.top_most
- .by_path(namespace_path)
+ namespace =
+ if Feature.enabled?(:npm_allow_packages_in_multiple_projects)
+ top_namespace_from(package_name)
+ else
+ namespace_path = ::Packages::Npm.scope_of(package_name)
+ next unless namespace_path
+
+ Namespace.top_most.by_path(namespace_path)
+ end
+
next unless namespace
finder = ::Packages::Npm::PackageFinder.new(
@@ -67,6 +82,15 @@ module API
end
end
end
+
+ private
+
+ def top_namespace_from(package_name)
+ namespace_path = ::Packages::Npm.scope_of(package_name)
+ return unless namespace_path
+
+ Namespace.top_most.by_path(namespace_path)
+ end
end
end
end
diff --git a/lib/api/helpers/packages_helpers.rb b/lib/api/helpers/packages_helpers.rb
index 0fb3a19b8fd..be2b73e2d48 100644
--- a/lib/api/helpers/packages_helpers.rb
+++ b/lib/api/helpers/packages_helpers.rb
@@ -90,6 +90,7 @@ module API
service.execute
category = args.delete(:category) || self.options[:for].name
+ args[:user] = current_user if current_user
event_name = "i_package_#{scope}_user"
::Gitlab::Tracking.event(
category,
@@ -100,8 +101,6 @@ module API
**args
)
- return unless Feature.enabled?(:route_hll_to_snowplow_phase3)
-
if action.to_s == 'push_package' && service.originator_type == :deploy_token
track_snowplow_event("push_package_by_deploy_token", category, args)
elsif action.to_s == 'pull_package' && service.originator_type == :guest
diff --git a/lib/api/helpers/projects_helpers.rb b/lib/api/helpers/projects_helpers.rb
index 2700ea90d59..b96e8efba61 100644
--- a/lib/api/helpers/projects_helpers.rb
+++ b/lib/api/helpers/projects_helpers.rb
@@ -32,7 +32,6 @@ module API
optional :builds_access_level, type: String, values: %w(disabled private enabled), desc: 'Builds access level. One of `disabled`, `private` or `enabled`'
optional :snippets_access_level, type: String, values: %w(disabled private enabled), desc: 'Snippets access level. One of `disabled`, `private` or `enabled`'
optional :pages_access_level, type: String, values: %w(disabled private enabled public), desc: 'Pages access level. One of `disabled`, `private`, `enabled` or `public`'
- optional :operations_access_level, type: String, values: %w(disabled private enabled), desc: 'Operations access level. One of `disabled`, `private` or `enabled`. Deprecated in GitLab 15.8, see https://gitlab.com/gitlab-org/gitlab/-/issues/385798.'
optional :analytics_access_level, type: String, values: %w(disabled private enabled), desc: 'Analytics access level. One of `disabled`, `private` or `enabled`'
optional :container_registry_access_level, type: String, values: %w(disabled private enabled), desc: 'Controls visibility of the container registry. One of `disabled`, `private` or `enabled`. `private` will make the container registry accessible only to project members (reporter role and above). `enabled` will make the container registry accessible to everyone who has access to the project. `disabled` will disable the container registry'
optional :security_and_compliance_access_level, type: String, values: %w(disabled private enabled), desc: 'Security and compliance access level. One of `disabled`, `private` or `enabled`'
@@ -157,7 +156,6 @@ module API
:name,
:only_allow_merge_if_all_discussions_are_resolved,
:only_allow_merge_if_pipeline_succeeds,
- :operations_access_level,
:pages_access_level,
:path,
:printing_merge_request_link_enabled,
diff --git a/lib/api/import_github.rb b/lib/api/import_github.rb
index 6330a4458f3..6550808a563 100644
--- a/lib/api/import_github.rb
+++ b/lib/api/import_github.rb
@@ -111,8 +111,6 @@ module API
requires :personal_access_token, type: String, desc: 'GitHub personal access token'
end
post 'import/github/gists' do
- not_found! if Feature.disabled?(:github_import_gists)
-
authorize! :create_snippet
result = Import::Github::GistsImportService.new(current_user, client, access_params).execute
diff --git a/lib/api/integrations.rb b/lib/api/integrations.rb
index 408fa038b0d..3ec0a723808 100644
--- a/lib/api/integrations.rb
+++ b/lib/api/integrations.rb
@@ -37,7 +37,7 @@ module API
end
end
- TRIGGER_INTEGRATIONS = {
+ SLASH_COMMAND_INTEGRATIONS = {
'mattermost-slash-commands' => [
{
name: :token,
@@ -139,8 +139,14 @@ module API
destroy_conditionally!(integration) do
attrs = integration_attributes(integration).index_with do |attr|
- column = integration.column_for_attribute(attr)
- if column.is_a?(ActiveRecord::ConnectionAdapters::NullColumn)
+ column = if integration.attribute_present?(attr)
+ integration.column_for_attribute(attr)
+ elsif integration.data_fields_present?
+ integration.data_fields.column_for_attribute(attr)
+ end
+
+ case column
+ when nil, ActiveRecord::ConnectionAdapters::NullColumn
nil
else
column.default
@@ -173,7 +179,7 @@ module API
end
end
- TRIGGER_INTEGRATIONS.each do |integration_slug, settings|
+ SLASH_COMMAND_INTEGRATIONS.each do |integration_slug, settings|
helpers do
def slash_command_integration(project, integration_slug, params)
project.integrations.active.find do |integration|
@@ -218,7 +224,27 @@ module API
end
end
end
+
+ desc "Trigger a global slack command" do
+ detail 'Added in GitLab 9.4'
+ failure [
+ { code: 401, message: 'Unauthorized' }
+ ]
+ end
+ params do
+ requires :text, type: String, desc: 'Text of the slack command'
+ end
+ post 'slack/trigger' do
+ if result = Gitlab::SlashCommands::GlobalSlackHandler.new(params).trigger
+ status result[:status] || 200
+ present result
+ else
+ not_found!
+ end
+ end
end
end
-API::Integrations.prepend_mod_with('API::Integrations')
+# Added for JiHu
+# https://gitlab.com/gitlab-org/gitlab/-/merge_requests/118289#note_1379334692
+API::Integrations.prepend_mod
diff --git a/lib/api/integrations/slack/concerns/verifies_request.rb b/lib/api/integrations/slack/concerns/verifies_request.rb
new file mode 100644
index 00000000000..b96f02e4cde
--- /dev/null
+++ b/lib/api/integrations/slack/concerns/verifies_request.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module API
+ class Integrations
+ module Slack
+ module Concerns
+ module VerifiesRequest
+ extend ActiveSupport::Concern
+
+ included do
+ before { verify_slack_request! }
+
+ helpers do
+ def verify_slack_request!
+ unauthorized! unless Request.verify!(request)
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/api/integrations/slack/events.rb b/lib/api/integrations/slack/events.rb
new file mode 100644
index 00000000000..7a0687a06b6
--- /dev/null
+++ b/lib/api/integrations/slack/events.rb
@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+# This API endpoint handles all events sent from Slack once a Slack
+# workspace has installed the GitLab Slack app.
+#
+# See https://api.slack.com/apis/connections/events-api.
+module API
+ class Integrations
+ module Slack
+ class Events < ::API::Base
+ include Slack::Concerns::VerifiesRequest
+
+ feature_category :integrations
+
+ namespace 'integrations/slack' do
+ desc 'Receive Slack events' do
+ success [
+ { code: 200, message: 'Successfully processed event' },
+ { code: 204, message: 'Failed to process event' }
+ ]
+ failure [
+ { code: 401, message: 'Unauthorized' }
+ ]
+ end
+
+ # Params are based on the JSON schema spec for Slack events https://api.slack.com/types/event.
+ # We mark all params as `optional` as we never want to fail a request from Slack. Slack may remove
+ # deprecated params in future that are currently described in their JSON schema spec as required.
+ params do
+ optional :token, type: String, desc: '(Deprecated by Slack) The request token, unused by GitLab'
+ optional :team_id, type: String, desc: 'The Slack workspace ID of where the event occurred'
+ optional :api_app_id, type: String, desc: 'The Slack app ID'
+ optional :event, type: Hash, desc: 'The event object with variable properties'
+ optional :type, type: String, desc: 'The kind of event this is, usually `event_callback`'
+ optional :event_id, type: String, desc: 'A unique identifier for this specific event'
+ optional :event_time, type: Integer, desc: 'The epoch timestamp in seconds when this event was dispatched'
+ optional :authed_users, type: Array[String], desc: '(Deprecated by Slack) An array of Slack user IDs'
+ end
+
+ post :events do
+ response = ::Integrations::SlackEventService.new(params).execute
+
+ status :ok
+
+ response.payload
+ rescue StandardError => e
+ # Track the error, but respond with a `2xx` because we don't want to risk
+ # Slack rate-limiting, or disabling our app, due to error responses.
+ # See https://api.slack.com/apis/connections/events-api.
+ Gitlab::ErrorTracking.track_exception(e)
+
+ no_content!
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/api/integrations/slack/interactions.rb b/lib/api/integrations/slack/interactions.rb
new file mode 100644
index 00000000000..af8331977f3
--- /dev/null
+++ b/lib/api/integrations/slack/interactions.rb
@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module API
+ # This API endpoint handles interaction payloads sent from Slack.
+ # See https://api.slack.com/interactivity/handling.
+ class Integrations
+ module Slack
+ class Interactions < ::API::Base
+ include Slack::Concerns::VerifiesRequest
+
+ feature_category :integrations
+
+ namespace 'integrations/slack' do
+ post :interactions, urgency: :low do
+ service_params = Gitlab::Json.parse(params[:payload]).deep_symbolize_keys!
+ response = ::Integrations::SlackInteractionService.new(service_params).execute
+
+ status :ok
+
+ response.payload
+ rescue StandardError => e
+ Gitlab::ErrorTracking.track_exception(e)
+
+ no_content!
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/api/integrations/slack/options.rb b/lib/api/integrations/slack/options.rb
new file mode 100644
index 00000000000..58e61b8cee0
--- /dev/null
+++ b/lib/api/integrations/slack/options.rb
@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module API
+ # This API endpoint handles options payloads sent from Slack.
+ # See https://api.slack.com/reference/block-kit/block-elements#external_select.
+ class Integrations
+ module Slack
+ class Options < ::API::Base
+ include Slack::Concerns::VerifiesRequest
+
+ feature_category :integrations
+
+ namespace 'integrations/slack' do
+ post :options, urgency: :low do
+ service_params = Gitlab::Json.parse(params[:payload]).deep_symbolize_keys!
+ response = ::Integrations::SlackOptionService.new(service_params).execute
+
+ status :ok
+
+ response.payload
+ rescue StandardError => e
+ Gitlab::ErrorTracking.track_exception(e)
+
+ no_content!
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/api/integrations/slack/request.rb b/lib/api/integrations/slack/request.rb
new file mode 100644
index 00000000000..df0109b07aa
--- /dev/null
+++ b/lib/api/integrations/slack/request.rb
@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module API
+ class Integrations
+ module Slack
+ module Request
+ VERIFICATION_VERSION = 'v0'
+ VERIFICATION_TIMESTAMP_HEADER = 'X-Slack-Request-Timestamp'
+ VERIFICATION_SIGNATURE_HEADER = 'X-Slack-Signature'
+ VERIFICATION_DELIMITER = ':'
+ VERIFICATION_HMAC_ALGORITHM = 'sha256'
+ VERIFICATION_TIMESTAMP_EXPIRY = 1.minute.to_i
+
+ # Verify the request by comparing the given request signature in the header
+ # with a signature value that we compute according to the steps in:
+ # https://api.slack.com/authentication/verifying-requests-from-slack.
+ def self.verify!(request)
+ return false unless Gitlab::CurrentSettings.slack_app_signing_secret
+
+ timestamp, signature = request.headers.values_at(
+ VERIFICATION_TIMESTAMP_HEADER,
+ VERIFICATION_SIGNATURE_HEADER
+ )
+
+ return false if timestamp.nil? || signature.nil?
+ return false if Time.current.to_i - timestamp.to_i >= VERIFICATION_TIMESTAMP_EXPIRY
+
+ request.body.rewind
+
+ basestring = [
+ VERIFICATION_VERSION,
+ timestamp,
+ request.body.read
+ ].join(VERIFICATION_DELIMITER)
+
+ hmac_digest = OpenSSL::HMAC.hexdigest(
+ VERIFICATION_HMAC_ALGORITHM,
+ Gitlab::CurrentSettings.slack_app_signing_secret,
+ basestring
+ )
+
+ # Signature will look like: 'v0=a2114d57b48eac39b9ad189dd8316235a7b4a8d21a10bd27519666489c69b503'
+ ActiveSupport::SecurityUtils.secure_compare(
+ signature,
+ "#{VERIFICATION_VERSION}=#{hmac_digest}"
+ )
+ end
+ end
+ end
+ end
+end
diff --git a/lib/api/internal/base.rb b/lib/api/internal/base.rb
index 3f6e052f7b6..2a5ff257718 100644
--- a/lib/api/internal/base.rb
+++ b/lib/api/internal/base.rb
@@ -195,7 +195,7 @@ module API
#
# Discover user by ssh key, user id or username
#
- get '/discover', feature_category: :authentication_and_authorization do
+ get '/discover', feature_category: :system_access do
present actor.user, with: Entities::UserSafe
end
@@ -208,7 +208,7 @@ module API
}
end
- post '/two_factor_recovery_codes', feature_category: :authentication_and_authorization do
+ post '/two_factor_recovery_codes', feature_category: :system_access do
status 200
actor.update_last_used_at!
@@ -237,7 +237,7 @@ module API
{ success: true, recovery_codes: codes }
end
- post '/personal_access_token', feature_category: :authentication_and_authorization do
+ post '/personal_access_token', feature_category: :system_access do
status 200
actor.update_last_used_at!
@@ -308,7 +308,7 @@ module API
# decided to pursue a different approach, so it's currently not used.
# We might revive the PAM module though as it provides better user
# flow.
- post '/two_factor_config', feature_category: :authentication_and_authorization do
+ post '/two_factor_config', feature_category: :system_access do
status 200
break { success: false } unless Feature.enabled?(:two_factor_for_cli)
@@ -330,13 +330,13 @@ module API
end
end
- post '/two_factor_push_otp_check', feature_category: :authentication_and_authorization do
+ post '/two_factor_push_otp_check', feature_category: :system_access do
status 200
two_factor_push_otp_check
end
- post '/two_factor_manual_otp_check', feature_category: :authentication_and_authorization do
+ post '/two_factor_manual_otp_check', feature_category: :system_access do
status 200
two_factor_manual_otp_check
diff --git a/lib/api/internal/kubernetes.rb b/lib/api/internal/kubernetes.rb
index 5f12275b7a0..d340e097700 100644
--- a/lib/api/internal/kubernetes.rb
+++ b/lib/api/internal/kubernetes.rb
@@ -47,7 +47,7 @@ module API
end
def check_feature_enabled
- not_found! unless Feature.enabled?(:kubernetes_agent_internal_api, type: :ops)
+ not_found!('Internal API not found') unless Feature.enabled?(:kubernetes_agent_internal_api, type: :ops)
end
def check_agent_token
@@ -73,6 +73,11 @@ module API
Gitlab::UsageDataCounters::KubernetesAgentCounter.increment_event_counts(events)
end
+
+ def update_configuration(agent:, config:)
+ ::Clusters::Agents::Authorizations::CiAccess::RefreshService.new(agent, config: config).execute
+ ::Clusters::Agents::Authorizations::UserAccess::RefreshService.new(agent, config: config).execute
+ end
end
namespace 'internal' do
@@ -85,7 +90,7 @@ module API
detail 'Retrieves agent info for the given token'
end
route_setting :authentication, cluster_agent_token_allowed: true
- get '/agent_info', feature_category: :kubernetes_management, urgency: :low do
+ get '/agent_info', feature_category: :deployment_management, urgency: :low do
project = agent.project
status 200
@@ -103,7 +108,7 @@ module API
detail 'Retrieves project info (if authorized)'
end
route_setting :authentication, cluster_agent_token_allowed: true
- get '/project_info', feature_category: :kubernetes_management, urgency: :low do
+ get '/project_info', feature_category: :deployment_management, urgency: :low do
project = find_project(params[:id])
not_found! unless agent_has_access_to_project?(project)
@@ -126,15 +131,57 @@ module API
requires :agent_id, type: Integer, desc: 'ID of the configured Agent'
requires :agent_config, type: JSON, desc: 'Configuration for the Agent'
end
- post '/', feature_category: :kubernetes_management, urgency: :low do
+ post '/', feature_category: :deployment_management, urgency: :low do
agent = ::Clusters::Agent.find(params[:agent_id])
-
- ::Clusters::Agents::RefreshAuthorizationService.new(agent, config: params[:agent_config]).execute
+ update_configuration(agent: agent, config: params[:agent_config])
no_content!
end
end
+ namespace 'kubernetes/authorize_proxy_user' do
+ desc 'Authorize a proxy user request'
+ params do
+ requires :agent_id, type: Integer, desc: 'ID of the agent accessed'
+ requires :access_type, type: String, values: ['session_cookie'], desc: 'The type of the access key being verified.'
+ requires :access_key, type: String, desc: 'The authentication secret for the given access type.'
+ given access_type: ->(val) { val == 'session_cookie' } do
+ requires :csrf_token, type: String, allow_blank: false, desc: 'CSRF token that must be checked when access_type is "session_cookie", to ensure the request originates from a GitLab browsing session.'
+ end
+ end
+ post '/', feature_category: :deployment_management do
+ # Load session
+ public_session_id_string =
+ begin
+ Gitlab::Kas::UserAccess.decrypt_public_session_id(params[:access_key])
+ rescue StandardError
+ bad_request!('Invalid access_key')
+ end
+
+ session_id = Rack::Session::SessionId.new(public_session_id_string)
+ session = ActiveSession.sessions_from_ids([session_id.private_id]).first
+ unauthorized!('Invalid session') unless session
+
+ # CSRF check
+ unless ::Gitlab::Kas::UserAccess.valid_authenticity_token?(session.symbolize_keys, params[:csrf_token])
+ unauthorized!('CSRF token does not match')
+ end
+
+ # Load user
+ user = Warden::SessionSerializer.new('rack.session' => session).fetch(:user)
+ unauthorized!('Invalid user in session') unless user
+
+ # Load agent
+ agent = ::Clusters::Agent.find(params[:agent_id])
+ unauthorized!('Feature disabled for agent') unless ::Gitlab::Kas::UserAccess.enabled_for?(agent)
+
+ service_response = ::Clusters::Agents::AuthorizeProxyUserService.new(user, agent).execute
+ render_api_error!(service_response[:message], service_response[:reason]) unless service_response.success?
+
+ service_response.payload
+ end
+ end
+
namespace 'kubernetes/usage_metrics' do
desc 'POST usage metrics' do
detail 'Updates usage metrics for agent'
@@ -149,7 +196,7 @@ module API
optional :agent_users_using_ci_tunnel, type: Array[Integer], desc: 'An array of user ids that have interacted with CI Tunnel'
end
end
- post '/', feature_category: :kubernetes_management do
+ post '/', feature_category: :deployment_management do
increment_count_events
increment_unique_events
diff --git a/lib/api/internal/pages.rb b/lib/api/internal/pages.rb
index 771059053ac..5664a3df589 100644
--- a/lib/api/internal/pages.rb
+++ b/lib/api/internal/pages.rb
@@ -33,26 +33,7 @@ module API
requires :host, type: String, desc: 'The host to query for'
end
get "/" do
- ##
- # Serverless domain proxy has been deprecated and disabled as per
- # https://gitlab.com/gitlab-org/gitlab-pages/-/issues/467
- #
- # serverless_domain_finder = ServerlessDomainFinder.new(params[:host])
- # if serverless_domain_finder.serverless?
- # # Handle Serverless domains
- # serverless_domain = serverless_domain_finder.execute
- # no_content! unless serverless_domain
- #
- # virtual_domain = Serverless::VirtualDomain.new(serverless_domain)
- # no_content! unless virtual_domain
- #
- # present virtual_domain, with: Entities::Internal::Serverless::VirtualDomain
- # end
-
- host = Namespace.find_by_pages_host(params[:host]) || PagesDomain.find_by_domain_case_insensitive(params[:host])
- no_content! unless host
-
- virtual_domain = host.pages_virtual_domain
+ virtual_domain = ::Gitlab::Pages::VirtualHostFinder.new(params[:host]).execute
no_content! unless virtual_domain
if virtual_domain.cache_key.present?
diff --git a/lib/api/issues.rb b/lib/api/issues.rb
index 88e99b29587..d033913aa71 100644
--- a/lib/api/issues.rb
+++ b/lib/api/issues.rb
@@ -99,7 +99,7 @@ module API
optional :add_labels, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, desc: 'Comma-separated list of label names'
optional :remove_labels, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, desc: 'Comma-separated list of label names'
optional :due_date, type: String, desc: 'Date string in the format YEAR-MONTH-DAY'
- optional :confidential, type: Boolean, desc: 'Boolean parameter if the issue should be confidential'
+ optional :confidential, type: Boolean, desc: 'Boolean parameter if the issue should be confidential', allow_blank: false
optional :discussion_locked, type: Boolean, desc: " Boolean parameter indicating if the issue's discussion is locked"
optional :issue_type, type: String, values: WorkItems::Type.allowed_types_for_issues, desc: "The type of the issue. Accepts: #{WorkItems::Type.allowed_types_for_issues.join(', ')}"
diff --git a/lib/api/keys.rb b/lib/api/keys.rb
index 77952bac01a..c711b3d9c19 100644
--- a/lib/api/keys.rb
+++ b/lib/api/keys.rb
@@ -5,7 +5,7 @@ module API
class Keys < ::API::Base
before { authenticate! }
- feature_category :authentication_and_authorization
+ feature_category :system_access
resource :keys do
desc 'Get single ssh key by id. Only available to admin users' do
diff --git a/lib/api/lint.rb b/lib/api/lint.rb
index 89787ba00c2..dee04b6bb00 100644
--- a/lib/api/lint.rb
+++ b/lib/api/lint.rb
@@ -2,20 +2,20 @@
module API
class Lint < ::API::Base
- feature_category :pipeline_authoring
+ feature_category :pipeline_composition
helpers do
def can_lint_ci?
signup_unrestricted = Gitlab::CurrentSettings.signup_enabled? && !Gitlab::CurrentSettings.signup_limited?
internal_user = current_user.present? && !current_user.external?
- is_developer = current_user.present? && current_user.projects.any? { |p| p.team.member?(current_user, Gitlab::Access::DEVELOPER) }
+ is_developer = current_user.present? && current_user.projects.any? { |p| p.member?(current_user, Gitlab::Access::DEVELOPER) }
signup_unrestricted || internal_user || is_developer
end
end
namespace :ci do
- desc 'Validates the .gitlab-ci.yml content' do
+ desc 'REMOVED: Validates the .gitlab-ci.yml content' do
detail 'Checks if CI/CD YAML configuration is valid'
success code: 200, model: Entities::Ci::Lint::Result
tags %w[ci_lint]
@@ -28,16 +28,7 @@ module API
end
post '/lint', urgency: :low do
- unauthorized! unless can_lint_ci?
-
- result = Gitlab::Ci::Lint.new(project: nil, current_user: current_user)
- .validate(params[:content], dry_run: false)
-
- status 200
- Entities::Ci::Lint::Result.represent(result, current_user: current_user, include_jobs: params[:include_jobs]).serializable_hash.tap do |presented_result|
- presented_result[:status] = presented_result[:valid] ? 'valid' : 'invalid'
- presented_result.delete(:merged_yaml) unless params[:include_merged_yaml]
- end
+ render_api_error!('410 Gone', 410)
end
end
@@ -56,7 +47,7 @@ module API
end
get ':id/ci/lint', urgency: :low do
- authorize! :read_code, user_project
+ authorize_read_code!
if user_project.commit.present?
content = user_project.repository.gitlab_ci_yml_for(user_project.commit.id, user_project.ci_config_path_or_default)
diff --git a/lib/api/maven_packages.rb b/lib/api/maven_packages.rb
index 411a53a481b..e075a917fa9 100644
--- a/lib/api/maven_packages.rb
+++ b/lib/api/maven_packages.rb
@@ -23,8 +23,14 @@ module API
helpers ::API::Helpers::PackagesHelpers
helpers ::API::Helpers::Packages::DependencyProxyHelpers
+ helpers ::API::Helpers::Packages::Maven::BasicAuthHelpers
helpers do
+ params :path_and_file_name do
+ requires :path, type: String, desc: 'Package path', documentation: { example: 'foo/bar/mypkg/1.0-SNAPSHOT' }
+ requires :file_name, type: String, desc: 'Package file name', documentation: { example: 'mypkg-1.0-SNAPSHOT.jar' }
+ end
+
def path_exists?(path)
return false if path.blank?
@@ -159,10 +165,9 @@ module API
tags %w[maven_packages]
end
params do
- requires :path, type: String, desc: 'Package path', documentation: { example: 'foo/bar/mypkg/1.0-SNAPSHOT' }
- requires :file_name, type: String, desc: 'Package file name', documentation: { example: 'mypkg-1.0-SNAPSHOT.jar' }
+ use :path_and_file_name
end
- route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true
+ route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true, basic_auth_personal_access_token: true
get 'packages/maven/*path/:file_name', requirements: MAVEN_ENDPOINT_REQUIREMENTS do
# return a similar failure to authorize_read_package!(project)
@@ -191,6 +196,7 @@ module API
package_file.file_sha1
else
track_package_event('pull_package', :maven, project: project, namespace: project.namespace) if jar_file?(format)
+
present_carrierwave_file_with_head_support!(package_file)
end
end
@@ -213,13 +219,12 @@ module API
end
resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
params do
- requires :path, type: String, desc: 'Package path', documentation: { example: 'foo/bar/mypkg/1.0-SNAPSHOT' }
- requires :file_name, type: String, desc: 'Package file name', documentation: { example: 'mypkg-1.0-SNAPSHOT.jar' }
+ use :path_and_file_name
end
- route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true
+ route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true, basic_auth_personal_access_token: true
get ':id/-/packages/maven/*path/:file_name', requirements: MAVEN_ENDPOINT_REQUIREMENTS do
# return a similar failure to group = find_group(params[:id])
- group = find_group(params[:id])
+ group = find_authorized_group!
if Feature.disabled?(:maven_central_request_forwarding, group&.root_ancestor)
not_found!('Group') unless path_exists?(params[:path])
@@ -240,7 +245,7 @@ module API
requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project'
end
resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
- desc 'Download the maven package file' do
+ desc 'Download the maven package file at a project level' do
detail 'This feature was introduced in GitLab 11.3'
success [
{ code: 200 },
@@ -254,12 +259,11 @@ module API
tags %w[maven_packages]
end
params do
- requires :path, type: String, desc: 'Package path', documentation: { example: 'foo/bar/mypkg/1.0-SNAPSHOT' }
- requires :file_name, type: String, desc: 'Package file name', documentation: { example: 'mypkg-1.0-SNAPSHOT.jar' }
+ use :path_and_file_name
end
- route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true
+ route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true, basic_auth_personal_access_token: true
get ':id/packages/maven/*path/:file_name', requirements: MAVEN_ENDPOINT_REQUIREMENTS do
- project = user_project(action: :read_package)
+ project = authorized_user_project(action: :read_package)
# return a similar failure to user_project
unless Feature.enabled?(:maven_central_request_forwarding, project&.root_ancestor)
@@ -318,42 +322,43 @@ module API
end
route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true
put ':id/packages/maven/*path/:file_name', requirements: MAVEN_ENDPOINT_REQUIREMENTS do
- unprocessable_entity! if Gitlab::FIPS.enabled? && params['file.md5']
+ unprocessable_entity! if Gitlab::FIPS.enabled? && params[:file].md5
authorize_upload!
bad_request!('File is too large') if user_project.actual_limits.exceeded?(:maven_max_file_size, params[:file].size)
file_name, format = extract_format(params[:file_name])
- result = ::Packages::Maven::FindOrCreatePackageService
- .new(user_project, current_user, params.merge(build: current_authenticated_job)).execute
+ ::Gitlab::Database::LoadBalancing::Session.current.use_primary do
+ result = ::Packages::Maven::FindOrCreatePackageService
+ .new(user_project, current_user, params.merge(build: current_authenticated_job)).execute
- bad_request!(result.errors.first) if result.error?
+ bad_request!(result.errors.first) if result.error?
- package = result.payload[:package]
+ package = result.payload[:package]
- case format
- when 'sha1'
- # After uploading a file, Maven tries to upload a sha1 and md5 version of it.
- # Since we store md5/sha1 in database we simply need to validate our hash
- # against one uploaded by Maven. We do this for `sha1` format.
- package_file = ::Packages::PackageFileFinder
- .new(package, file_name).execute!
+ case format
+ when 'sha1'
+ # After uploading a file, Maven tries to upload a sha1 and md5 version of it.
+ # Since we store md5/sha1 in database we simply need to validate our hash
+ # against one uploaded by Maven. We do this for `sha1` format.
+ package_file = ::Packages::PackageFileFinder
+ .new(package, file_name).execute!
- verify_package_file(package_file, params[:file])
- when 'md5'
- ''
- else
- file_params = {
- file: params[:file],
- size: params['file.size'],
- file_name: file_name,
- file_type: params['file.type'],
- file_sha1: params['file.sha1'],
- file_md5: params['file.md5']
- }
-
- ::Packages::CreatePackageFileService.new(package, file_params.merge(build: current_authenticated_job)).execute
- track_package_event('push_package', :maven, user: current_user, project: user_project, namespace: user_project.namespace) if jar_file?(format)
+ verify_package_file(package_file, params[:file])
+ when 'md5'
+ ''
+ else
+ file_params = {
+ file: params[:file],
+ size: params[:file].size,
+ file_name: file_name,
+ file_sha1: params[:file].sha1,
+ file_md5: params[:file].md5
+ }
+
+ ::Packages::CreatePackageFileService.new(package, file_params.merge(build: current_authenticated_job)).execute
+ track_package_event('push_package', :maven, project: user_project, namespace: user_project.namespace) if jar_file?(format)
+ end
end
end
end
diff --git a/lib/api/members.rb b/lib/api/members.rb
index 1e640a6542a..9321b7ad8d5 100644
--- a/lib/api/members.rb
+++ b/lib/api/members.rb
@@ -177,6 +177,8 @@ module API
source = find_source(source_type, params[:id])
member = source_members(source).find_by!(user_id: params[:user_id])
+ check_rate_limit!(:member_delete, scope: [source, current_user])
+
destroy_conditionally!(member) do
::Members::DestroyService.new(current_user).execute(member, skip_subresources: params[:skip_subresources], unassign_issuables: params[:unassign_issuables])
end
diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb
index cd46b442b68..c29a7eee923 100644
--- a/lib/api/merge_requests.rb
+++ b/lib/api/merge_requests.rb
@@ -8,6 +8,10 @@ module API
before { authenticate_non_get! }
+ rescue_from ActiveRecord::QueryCanceled do |_e|
+ render_api_error!({ error: 'Request timed out' }, 408)
+ end
+
helpers Helpers::MergeRequestsHelpers
# These endpoints are defined in `TimeTrackingEndpoints` and is shared by
@@ -114,6 +118,9 @@ module API
end
def recheck_mergeability_of(merge_requests:)
+ return if ::Feature.enabled?(:restrict_merge_status_recheck, user_project) &&
+ !can?(current_user, :update_merge_request, user_project)
+
merge_requests.each { |mr| mr.check_mergeability(async: true) }
end
diff --git a/lib/api/metrics/dashboard/annotations.rb b/lib/api/metrics/dashboard/annotations.rb
index 6ba154191be..6edf4783159 100644
--- a/lib/api/metrics/dashboard/annotations.rb
+++ b/lib/api/metrics/dashboard/annotations.rb
@@ -40,9 +40,11 @@ module API
end
post ':id/metrics_dashboard/annotations' do
+ not_found! if Feature.enabled?(:remove_monitor_metrics)
+
annotations_source_object = annotations_source[:class].find(params[:id])
- forbidden! unless can?(current_user, :create_metrics_dashboard_annotation, annotations_source_object)
+ forbidden! unless can?(current_user, :admin_metrics_dashboard_annotation, annotations_source_object)
create_service_params = declared(params).merge(
annotations_source[:create_service_param_key] => annotations_source_object
diff --git a/lib/api/metrics/user_starred_dashboards.rb b/lib/api/metrics/user_starred_dashboards.rb
index 0a91e914d52..b7fba2b6459 100644
--- a/lib/api/metrics/user_starred_dashboards.rb
+++ b/lib/api/metrics/user_starred_dashboards.rb
@@ -25,6 +25,8 @@ module API
end
post ':id/metrics/user_starred_dashboards' do
+ not_found! if Feature.enabled?(:remove_monitor_metrics)
+
result = ::Metrics::UsersStarredDashboards::CreateService.new(current_user, user_project, params[:dashboard_path]).execute
if result.success?
@@ -50,6 +52,8 @@ module API
end
delete ':id/metrics/user_starred_dashboards' do
+ not_found! if Feature.enabled?(:remove_monitor_metrics)
+
result = ::Metrics::UsersStarredDashboards::DeleteService.new(current_user, user_project, params[:dashboard_path]).execute
if result.success?
diff --git a/lib/api/milestone_responses.rb b/lib/api/milestone_responses.rb
index 2fd3239b44a..fb71cb0e791 100644
--- a/lib/api/milestone_responses.rb
+++ b/lib/api/milestone_responses.rb
@@ -20,6 +20,8 @@ module API
optional :search, type: String, desc: 'The search criteria for the title or description of the milestone'
optional :include_parent_milestones, type: Grape::API::Boolean, default: false,
desc: 'Include group milestones from parent and its ancestors'
+ optional :updated_before, type: DateTime, desc: 'Return milestones updated before the specified datetime. Format: ISO 8601 YYYY-MM-DDTHH:MM:SSZ'
+ optional :updated_after, type: DateTime, desc: 'Return milestones updated after the specified datetime. Format: ISO 8601 YYYY-MM-DDTHH:MM:SSZ'
use :pagination
end
@@ -33,14 +35,9 @@ module API
end
def list_milestones_for(parent)
- milestones = init_milestones_collection(parent)
- milestones = Milestone.filter_by_state(milestones, params[:state])
- if params[:iids].present? && !params[:include_parent_milestones]
- milestones = filter_by_iid(milestones, params[:iids])
- end
-
- milestones = filter_by_title(milestones, params[:title]) if params[:title]
- milestones = filter_by_search(milestones, params[:search]) if params[:search]
+ milestones = MilestonesFinder.new(
+ params.merge(parent_finder_params(parent))
+ ).execute
present paginate(milestones), with: Entities::Milestone
end
@@ -84,6 +81,16 @@ module API
present paginate(issuables), with: entity, current_user: current_user
end
+ def parent_finder_params(parent)
+ include_parent = params[:include_parent_milestones].present?
+
+ if parent.is_a?(Project)
+ { project_ids: parent.id, group_ids: (include_parent ? project_group_ids(parent) : nil) }
+ else
+ { group_ids: (include_parent ? group_and_ancestor_ids(parent) : parent.id) }
+ end
+ end
+
def build_finder_params(milestone, parent)
finder_params = { milestone_title: milestone.title, sort: 'label_priority' }
@@ -102,26 +109,6 @@ module API
end
end
- def init_milestones_collection(parent)
- milestones = if params[:include_parent_milestones].present?
- parent_and_ancestors_milestones(parent)
- else
- parent.milestones
- end
-
- milestones.order_id_desc
- end
-
- def parent_and_ancestors_milestones(parent)
- project_id, group_ids = if parent.is_a?(Project)
- [parent.id, project_group_ids(parent)]
- else
- [nil, parent_group_ids(parent)]
- end
-
- Milestone.for_projects_and_groups(project_id, group_ids)
- end
-
def project_group_ids(parent)
group = parent.group
return unless group.present?
@@ -129,7 +116,7 @@ module API
group.self_and_ancestors.select(:id)
end
- def parent_group_ids(group)
+ def group_and_ancestor_ids(group)
return unless group.present?
group.self_and_ancestors
diff --git a/lib/api/ml/mlflow.rb b/lib/api/ml/mlflow.rb
deleted file mode 100644
index e7ed8e2e70c..00000000000
--- a/lib/api/ml/mlflow.rb
+++ /dev/null
@@ -1,297 +0,0 @@
-# frozen_string_literal: true
-
-require 'mime/types'
-
-module API
- # MLFlow integration API, replicating the Rest API https://www.mlflow.org/docs/latest/rest-api.html#rest-api
- module Ml
- class Mlflow < ::API::Base
- include APIGuard
-
- # The first part of the url is the namespace, the second part of the URL is what the MLFlow client calls
- MLFLOW_API_PREFIX = ':id/ml/mlflow/api/2.0/mlflow/'
-
- allow_access_with_scope :api
- allow_access_with_scope :read_api, if: -> (request) { request.get? || request.head? }
-
- feature_category :mlops
-
- content_type :json, 'application/json'
- default_format :json
-
- before do
- # MLFlow Client considers any status code different than 200 an error, even 201
- status 200
-
- authenticate!
-
- not_found! unless Feature.enabled?(:ml_experiment_tracking, user_project)
- end
-
- rescue_from ActiveRecord::ActiveRecordError do |e|
- invalid_parameter!(e.message)
- end
-
- helpers do
- def resource_not_found!
- render_structured_api_error!({ error_code: 'RESOURCE_DOES_NOT_EXIST' }, 404)
- end
-
- def resource_already_exists!
- render_structured_api_error!({ error_code: 'RESOURCE_ALREADY_EXISTS' }, 400)
- end
-
- def invalid_parameter!(message = nil)
- render_structured_api_error!({ error_code: 'INVALID_PARAMETER_VALUE', message: message }, 400)
- end
-
- def experiment_repository
- ::Ml::ExperimentTracking::ExperimentRepository.new(user_project, current_user)
- end
-
- def candidate_repository
- ::Ml::ExperimentTracking::CandidateRepository.new(user_project, current_user)
- end
-
- def experiment
- @experiment ||= find_experiment!(params[:experiment_id], params[:experiment_name])
- end
-
- def candidate
- @candidate ||= find_candidate!(params[:run_id])
- end
-
- def find_experiment!(iid, name)
- experiment_repository.by_iid_or_name(iid: iid, name: name) || resource_not_found!
- end
-
- def find_candidate!(iid)
- candidate_repository.by_iid(iid) || resource_not_found!
- end
-
- def packages_url
- path = api_v4_projects_packages_generic_package_version_path(
- id: user_project.id, package_name: '', file_name: ''
- )
- path = path.delete_suffix('/package_version')
-
- "#{request.base_url}#{path}"
- end
- end
-
- params do
- requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project'
- end
- resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
- desc 'API to interface with MLFlow Client, REST API version 1.28.0' do
- detail 'This feature is gated by :ml_experiment_tracking.'
- end
- namespace MLFLOW_API_PREFIX do
- resource :experiments do
- desc 'Fetch experiment by experiment_id' do
- success Entities::Ml::Mlflow::GetExperiment
- detail 'https://www.mlflow.org/docs/1.28.0/rest-api.html#get-experiment'
- end
- params do
- optional :experiment_id, type: String, default: '', desc: 'Experiment ID, in reference to the project'
- end
- get 'get', urgency: :low do
- present experiment, with: Entities::Ml::Mlflow::GetExperiment
- end
-
- desc 'Fetch experiment by experiment_name' do
- success Entities::Ml::Mlflow::GetExperiment
- detail 'https://www.mlflow.org/docs/1.28.0/rest-api.html#get-experiment-by-name'
- end
- params do
- optional :experiment_name, type: String, default: '', desc: 'Experiment name'
- end
- get 'get-by-name', urgency: :low do
- present experiment, with: Entities::Ml::Mlflow::GetExperiment
- end
-
- desc 'List experiments' do
- success Entities::Ml::Mlflow::ListExperiment
- detail 'https://www.mlflow.org/docs/latest/rest-api.html#list-experiments'
- end
- get 'list', urgency: :low do
- response = { experiments: experiment_repository.all }
-
- present response, with: Entities::Ml::Mlflow::ListExperiment
- end
-
- desc 'Create experiment' do
- success Entities::Ml::Mlflow::NewExperiment
- detail 'https://www.mlflow.org/docs/1.28.0/rest-api.html#create-experiment'
- end
- params do
- requires :name, type: String, desc: 'Experiment name'
- optional :tags, type: Array, desc: 'Tags with information about the experiment'
- optional :artifact_location, type: String, desc: 'This will be ignored'
- end
- post 'create', urgency: :low do
- present experiment_repository.create!(params[:name], params[:tags]),
- with: Entities::Ml::Mlflow::NewExperiment
- rescue ActiveRecord::RecordInvalid
- resource_already_exists!
- end
-
- desc 'Sets a tag for an experiment.' do
- summary 'Sets a tag for an experiment. '
-
- detail 'https://www.mlflow.org/docs/1.28.0/rest-api.html#set-experiment-tag'
- end
- params do
- requires :experiment_id, type: String, desc: 'ID of the experiment.'
- requires :key, type: String, desc: 'Name for the tag.'
- requires :value, type: String, desc: 'Value for the tag.'
- end
- post 'set-experiment-tag', urgency: :low do
- bad_request! unless experiment_repository.add_tag!(experiment, params[:key], params[:value])
-
- {}
- end
- end
-
- resource :runs do
- desc 'Creates a Run.' do
- success Entities::Ml::Mlflow::Run
- detail 'MLFlow Runs map to GitLab Candidates. https://www.mlflow.org/docs/1.28.0/rest-api.html#create-run'
- end
- params do
- requires :experiment_id, type: Integer,
- desc: 'Id for the experiment, relative to the project'
- optional :start_time, type: Integer,
- desc: 'Unix timestamp in milliseconds of when the run started.',
- default: 0
- optional :user_id, type: String, desc: 'This will be ignored'
- optional :tags, type: Array, desc: 'Tags are stored, but not displayed'
- optional :run_name, type: String, desc: 'A name for this run'
- end
- post 'create', urgency: :low do
- present candidate_repository.create!(experiment, params[:start_time], params[:tags], params[:run_name]),
- with: Entities::Ml::Mlflow::Run, packages_url: packages_url
- end
-
- desc 'Gets an MLFlow Run, which maps to GitLab Candidates' do
- success Entities::Ml::Mlflow::Run
- detail 'https://www.mlflow.org/docs/1.28.0/rest-api.html#get-run'
- end
- params do
- requires :run_id, type: String, desc: 'UUID of the candidate.'
- optional :run_uuid, type: String, desc: 'This parameter is ignored'
- end
- get 'get', urgency: :low do
- present candidate, with: Entities::Ml::Mlflow::Run, packages_url: packages_url
- end
-
- desc 'Updates a Run.' do
- success Entities::Ml::Mlflow::UpdateRun
- detail 'MLFlow Runs map to GitLab Candidates. https://www.mlflow.org/docs/1.28.0/rest-api.html#update-run'
- end
- params do
- requires :run_id, type: String, desc: 'UUID of the candidate.'
- optional :status, type: String,
- values: ::Ml::Candidate.statuses.keys.map(&:upcase),
- desc: "Status of the run. Accepts: " \
- "#{::Ml::Candidate.statuses.keys.map(&:upcase)}."
- optional :end_time, type: Integer, desc: 'Ending time of the run'
- end
- post 'update', urgency: :low do
- candidate_repository.update(candidate, params[:status], params[:end_time])
-
- present candidate, with: Entities::Ml::Mlflow::UpdateRun, packages_url: packages_url
- end
-
- desc 'Logs a metric to a run.' do
- summary 'Log a metric for a run. A metric is a key-value pair (string key, float value) with an '\
- 'associated timestamp. Examples include the various metrics that represent ML model accuracy. '\
- 'A metric can be logged multiple times.'
- detail 'https://www.mlflow.org/docs/1.28.0/rest-api.html#log-metric'
- end
- params do
- requires :run_id, type: String, desc: 'UUID of the run.'
- requires :key, type: String, desc: 'Name for the metric.'
- requires :value, type: Float, desc: 'Value of the metric.'
- requires :timestamp, type: Integer, desc: 'Unix timestamp in milliseconds when metric was recorded'
- optional :step, type: Integer, desc: 'Step at which the metric was recorded'
- end
- post 'log-metric', urgency: :low do
- candidate_repository.add_metric!(
- candidate,
- params[:key],
- params[:value],
- params[:timestamp],
- params[:step]
- )
-
- {}
- end
-
- desc 'Logs a parameter to a run.' do
- summary 'Log a param used for a run. A param is a key-value pair (string key, string value). '\
- 'Examples include hyperparameters used for ML model training and constant dates and values '\
- 'used in an ETL pipeline. A param can be logged only once for a run, duplicate will be .'\
- 'ignored'
-
- detail 'https://www.mlflow.org/docs/1.28.0/rest-api.html#log-param'
- end
- params do
- requires :run_id, type: String, desc: 'UUID of the run.'
- requires :key, type: String, desc: 'Name for the parameter.'
- requires :value, type: String, desc: 'Value for the parameter.'
- end
- post 'log-parameter', urgency: :low do
- bad_request! unless candidate_repository.add_param!(candidate, params[:key], params[:value])
-
- {}
- end
-
- desc 'Sets a tag for a run.' do
- summary 'Sets a tag for a run. '
-
- detail 'https://www.mlflow.org/docs/1.28.0/rest-api.html#set-tag'
- end
- params do
- requires :run_id, type: String, desc: 'UUID of the run.'
- requires :key, type: String, desc: 'Name for the tag.'
- requires :value, type: String, desc: 'Value for the tag.'
- end
- post 'set-tag', urgency: :low do
- bad_request! unless candidate_repository.add_tag!(candidate, params[:key], params[:value])
-
- {}
- end
-
- desc 'Logs multiple parameters and metrics.' do
- summary 'Log a batch of metrics and params for a run. Validation errors will block the entire batch, '\
- 'duplicate errors will be ignored.'
-
- detail 'https://www.mlflow.org/docs/1.28.0/rest-api.html#log-param'
- end
- params do
- requires :run_id, type: String, desc: 'UUID of the run.'
- optional :metrics, type: Array, default: [] do
- requires :key, type: String, desc: 'Name for the metric.'
- requires :value, type: Float, desc: 'Value of the metric.'
- requires :timestamp, type: Integer, desc: 'Unix timestamp in milliseconds when metric was recorded'
- optional :step, type: Integer, desc: 'Step at which the metric was recorded'
- end
- optional :params, type: Array, default: [] do
- requires :key, type: String, desc: 'Name for the metric.'
- requires :value, type: String, desc: 'Value of the metric.'
- end
- end
- post 'log-batch', urgency: :low do
- candidate_repository.add_metrics(candidate, params[:metrics])
- candidate_repository.add_params(candidate, params[:params])
- candidate_repository.add_tags(candidate, params[:tags])
-
- {}
- end
- end
- end
- end
- end
- end
-end
diff --git a/lib/api/ml/mlflow/api_helpers.rb b/lib/api/ml/mlflow/api_helpers.rb
new file mode 100644
index 00000000000..7f4a895235c
--- /dev/null
+++ b/lib/api/ml/mlflow/api_helpers.rb
@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module API
+ module Ml
+ module Mlflow
+ module ApiHelpers
+ def resource_not_found!
+ render_structured_api_error!({ error_code: 'RESOURCE_DOES_NOT_EXIST' }, 404)
+ end
+
+ def resource_already_exists!
+ render_structured_api_error!({ error_code: 'RESOURCE_ALREADY_EXISTS' }, 400)
+ end
+
+ def invalid_parameter!(message = nil)
+ render_structured_api_error!({ error_code: 'INVALID_PARAMETER_VALUE', message: message }, 400)
+ end
+
+ def experiment_repository
+ ::Ml::ExperimentTracking::ExperimentRepository.new(user_project, current_user)
+ end
+
+ def candidate_repository
+ ::Ml::ExperimentTracking::CandidateRepository.new(user_project, current_user)
+ end
+
+ def experiment
+ @experiment ||= find_experiment!(params[:experiment_id], params[:experiment_name])
+ end
+
+ def candidate
+ @candidate ||= find_candidate!(params[:run_id])
+ end
+
+ def find_experiment!(iid, name)
+ experiment_repository.by_iid_or_name(iid: iid, name: name) || resource_not_found!
+ end
+
+ def find_candidate!(eid)
+ candidate_repository.by_eid(eid) || resource_not_found!
+ end
+
+ def packages_url
+ path = api_v4_projects_packages_generic_package_version_path(
+ id: user_project.id, package_name: '', file_name: ''
+ )
+ path = path.delete_suffix('/package_version')
+
+ "#{request.base_url}#{path}"
+ end
+ end
+ end
+ end
+end
diff --git a/lib/api/ml/mlflow/entrypoint.rb b/lib/api/ml/mlflow/entrypoint.rb
new file mode 100644
index 00000000000..880b1efeb5a
--- /dev/null
+++ b/lib/api/ml/mlflow/entrypoint.rb
@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module API
+ # MLFlow integration API, replicating the Rest API https://www.mlflow.org/docs/latest/rest-api.html#rest-api
+ module Ml
+ module Mlflow
+ class Entrypoint < ::API::Base
+ include APIGuard
+
+ # The first part of the url is the namespace, the second part of the URL is what the MLFlow client calls
+ MLFLOW_API_PREFIX = ':id/ml/mlflow/api/2.0/mlflow/'
+
+ helpers ::API::Ml::Mlflow::ApiHelpers
+
+ allow_access_with_scope :api
+ allow_access_with_scope :read_api, if: ->(request) { request.get? || request.head? }
+
+ feature_category :mlops
+
+ content_type :json, 'application/json'
+ default_format :json
+
+ before do
+ # MLFlow Client considers any status code different than 200 an error, even 201
+ status 200
+
+ authenticate!
+
+ not_found! unless Feature.enabled?(:ml_experiment_tracking, user_project)
+ end
+
+ rescue_from ActiveRecord::ActiveRecordError do |e|
+ invalid_parameter!(e.message)
+ end
+
+ params do
+ requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project'
+ end
+ resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
+ desc 'API to interface with MLFlow Client, REST API version 1.28.0' do
+ detail 'This feature is gated by :ml_experiment_tracking.'
+ end
+ namespace MLFLOW_API_PREFIX do
+ mount ::API::Ml::Mlflow::Experiments
+ mount ::API::Ml::Mlflow::Runs
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/api/ml/mlflow/experiments.rb b/lib/api/ml/mlflow/experiments.rb
new file mode 100644
index 00000000000..614112f703b
--- /dev/null
+++ b/lib/api/ml/mlflow/experiments.rb
@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+require 'mime/types'
+
+module API
+ # MLFlow integration API, replicating the Rest API https://www.mlflow.org/docs/latest/rest-api.html#rest-api
+ module Ml
+ module Mlflow
+ class Experiments < ::API::Base
+ feature_category :mlops
+
+ resource :experiments do
+ desc 'Fetch experiment by experiment_id' do
+ success Entities::Ml::Mlflow::GetExperiment
+ detail 'https://www.mlflow.org/docs/1.28.0/rest-api.html#get-experiment'
+ end
+ params do
+ optional :experiment_id, type: String, default: '', desc: 'Experiment ID, in reference to the project'
+ end
+ get 'get', urgency: :low do
+ present experiment, with: Entities::Ml::Mlflow::GetExperiment
+ end
+
+ desc 'Fetch experiment by experiment_name' do
+ success Entities::Ml::Mlflow::GetExperiment
+ detail 'https://www.mlflow.org/docs/1.28.0/rest-api.html#get-experiment-by-name'
+ end
+ params do
+ optional :experiment_name, type: String, default: '', desc: 'Experiment name'
+ end
+ get 'get-by-name', urgency: :low do
+ present experiment, with: Entities::Ml::Mlflow::GetExperiment
+ end
+
+ desc 'List experiments' do
+ success Entities::Ml::Mlflow::ListExperiment
+ detail 'https://www.mlflow.org/docs/latest/rest-api.html#list-experiments'
+ end
+ get 'list', urgency: :low do
+ response = { experiments: experiment_repository.all }
+
+ present response, with: Entities::Ml::Mlflow::ListExperiment
+ end
+
+ desc 'Create experiment' do
+ success Entities::Ml::Mlflow::NewExperiment
+ detail 'https://www.mlflow.org/docs/1.28.0/rest-api.html#create-experiment'
+ end
+ params do
+ requires :name, type: String, desc: 'Experiment name'
+ optional :tags, type: Array, desc: 'Tags with information about the experiment'
+ optional :artifact_location, type: String, desc: 'This will be ignored'
+ end
+ post 'create', urgency: :low do
+ present experiment_repository.create!(params[:name], params[:tags]),
+ with: Entities::Ml::Mlflow::NewExperiment
+ rescue ActiveRecord::RecordInvalid
+ resource_already_exists!
+ end
+
+ desc 'Sets a tag for an experiment.' do
+ summary 'Sets a tag for an experiment. '
+
+ detail 'https://www.mlflow.org/docs/1.28.0/rest-api.html#set-experiment-tag'
+ end
+ params do
+ requires :experiment_id, type: String, desc: 'ID of the experiment.'
+ requires :key, type: String, desc: 'Name for the tag.'
+ requires :value, type: String, desc: 'Value for the tag.'
+ end
+ post 'set-experiment-tag', urgency: :low do
+ bad_request! unless experiment_repository.add_tag!(experiment, params[:key], params[:value])
+
+ {}
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/api/ml/mlflow/runs.rb b/lib/api/ml/mlflow/runs.rb
new file mode 100644
index 00000000000..f737c6bd497
--- /dev/null
+++ b/lib/api/ml/mlflow/runs.rb
@@ -0,0 +1,152 @@
+# frozen_string_literal: true
+
+require 'mime/types'
+
+module API
+ # MLFlow integration API, replicating the Rest API https://www.mlflow.org/docs/latest/rest-api.html#rest-api
+ module Ml
+ module Mlflow
+ class Runs < ::API::Base
+ feature_category :mlops
+
+ resource :runs do
+ desc 'Creates a Run.' do
+ success Entities::Ml::Mlflow::Run
+ detail 'MLFlow Runs map to GitLab Candidates. https://www.mlflow.org/docs/1.28.0/rest-api.html#create-run'
+ end
+ params do
+ requires :experiment_id, type: Integer,
+ desc: 'Id for the experiment, relative to the project'
+ optional :start_time, type: Integer,
+ desc: 'Unix timestamp in milliseconds of when the run started.',
+ default: 0
+ optional :user_id, type: String, desc: 'This will be ignored'
+ optional :tags, type: Array, desc: 'Tags are stored, but not displayed'
+ optional :run_name, type: String, desc: 'A name for this run'
+ end
+ post 'create', urgency: :low do
+ present candidate_repository.create!(experiment, params[:start_time], params[:tags], params[:run_name]),
+ with: Entities::Ml::Mlflow::Run, packages_url: packages_url
+ end
+
+ desc 'Gets an MLFlow Run, which maps to GitLab Candidates' do
+ success Entities::Ml::Mlflow::Run
+ detail 'https://www.mlflow.org/docs/1.28.0/rest-api.html#get-run'
+ end
+ params do
+ requires :run_id, type: String, desc: 'UUID of the candidate.'
+ optional :run_uuid, type: String, desc: 'This parameter is ignored'
+ end
+ get 'get', urgency: :low do
+ present candidate, with: Entities::Ml::Mlflow::Run, packages_url: packages_url
+ end
+
+ desc 'Updates a Run.' do
+ success Entities::Ml::Mlflow::UpdateRun
+ detail 'MLFlow Runs map to GitLab Candidates. https://www.mlflow.org/docs/1.28.0/rest-api.html#update-run'
+ end
+ params do
+ requires :run_id, type: String, desc: 'UUID of the candidate.'
+ optional :status, type: String,
+ values: ::Ml::Candidate.statuses.keys.map(&:upcase),
+ desc: "Status of the run. Accepts: " \
+ "#{::Ml::Candidate.statuses.keys.map(&:upcase)}."
+ optional :end_time, type: Integer, desc: 'Ending time of the run'
+ end
+ post 'update', urgency: :low do
+ candidate_repository.update(candidate, params[:status], params[:end_time])
+
+ present candidate, with: Entities::Ml::Mlflow::UpdateRun, packages_url: packages_url
+ end
+
+ desc 'Logs a metric to a run.' do
+ summary 'Log a metric for a run. A metric is a key-value pair (string key, float value) with an ' \
+ 'associated timestamp. Examples include the various metrics that represent ML model accuracy. ' \
+ 'A metric can be logged multiple times.'
+ detail 'https://www.mlflow.org/docs/1.28.0/rest-api.html#log-metric'
+ end
+ params do
+ requires :run_id, type: String, desc: 'UUID of the run.'
+ requires :key, type: String, desc: 'Name for the metric.'
+ requires :value, type: Float, desc: 'Value of the metric.'
+ requires :timestamp, type: Integer, desc: 'Unix timestamp in milliseconds when metric was recorded'
+ optional :step, type: Integer, desc: 'Step at which the metric was recorded'
+ end
+ post 'log-metric', urgency: :low do
+ candidate_repository.add_metric!(
+ candidate,
+ params[:key],
+ params[:value],
+ params[:timestamp],
+ params[:step]
+ )
+
+ {}
+ end
+
+ desc 'Logs a parameter to a run.' do
+ summary 'Log a param used for a run. A param is a key-value pair (string key, string value). ' \
+ 'Examples include hyperparameters used for ML model training and constant dates and values ' \
+ 'used in an ETL pipeline. A param can be logged only once for a run, duplicate will be .' \
+ 'ignored'
+
+ detail 'https://www.mlflow.org/docs/1.28.0/rest-api.html#log-param'
+ end
+ params do
+ requires :run_id, type: String, desc: 'UUID of the run.'
+ requires :key, type: String, desc: 'Name for the parameter.'
+ requires :value, type: String, desc: 'Value for the parameter.'
+ end
+ post 'log-parameter', urgency: :low do
+ bad_request! unless candidate_repository.add_param!(candidate, params[:key], params[:value])
+
+ {}
+ end
+
+ desc 'Sets a tag for a run.' do
+ summary 'Sets a tag for a run. '
+
+ detail 'https://www.mlflow.org/docs/1.28.0/rest-api.html#set-tag'
+ end
+ params do
+ requires :run_id, type: String, desc: 'UUID of the run.'
+ requires :key, type: String, desc: 'Name for the tag.'
+ requires :value, type: String, desc: 'Value for the tag.'
+ end
+ post 'set-tag', urgency: :low do
+ bad_request! unless candidate_repository.add_tag!(candidate, params[:key], params[:value])
+
+ {}
+ end
+
+ desc 'Logs multiple parameters and metrics.' do
+ summary 'Log a batch of metrics and params for a run. Validation errors will block the entire batch, ' \
+ 'duplicate errors will be ignored.'
+
+ detail 'https://www.mlflow.org/docs/1.28.0/rest-api.html#log-param'
+ end
+ params do
+ requires :run_id, type: String, desc: 'UUID of the run.'
+ optional :metrics, type: Array, default: [] do
+ requires :key, type: String, desc: 'Name for the metric.'
+ requires :value, type: Float, desc: 'Value of the metric.'
+ requires :timestamp, type: Integer, desc: 'Unix timestamp in milliseconds when metric was recorded'
+ optional :step, type: Integer, desc: 'Step at which the metric was recorded'
+ end
+ optional :params, type: Array, default: [] do
+ requires :key, type: String, desc: 'Name for the metric.'
+ requires :value, type: String, desc: 'Value of the metric.'
+ end
+ end
+ post 'log-batch', urgency: :low do
+ candidate_repository.add_metrics(candidate, params[:metrics])
+ candidate_repository.add_params(candidate, params[:params])
+ candidate_repository.add_tags(candidate, params[:tags])
+
+ {}
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/api/npm_project_packages.rb b/lib/api/npm_project_packages.rb
index f42ded5ac09..171a061bf97 100644
--- a/lib/api/npm_project_packages.rb
+++ b/lib/api/npm_project_packages.rb
@@ -44,8 +44,8 @@ module API
present_package_file!(package_file)
end
- desc 'Create NPM package' do
- detail 'This feature was introduced in GitLab 11.8'
+ desc 'Create or deprecate NPM package' do
+ detail 'Create was introduced in GitLab 11.8 & deprecate suppport was added in 16.0'
success code: 200
failure [
{ code: 400, message: 'Bad Request' },
@@ -61,16 +61,22 @@ module API
end
route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true
put ':package_name', requirements: ::API::Helpers::Packages::Npm::NPM_ENDPOINT_REQUIREMENTS do
- authorize_create_package!(project)
-
- created_package = ::Packages::Npm::CreatePackageService
- .new(project, current_user, params.merge(build: current_authenticated_job)).execute
+ if headers['Npm-Command'] == 'deprecate'
+ authorize_destroy_package!(project)
- if created_package[:status] == :error
- render_api_error!(created_package[:message], created_package[:http_status])
+ ::Packages::Npm::DeprecatePackageService.new(project, declared(params)).execute(async: true)
else
- track_package_event('push_package', :npm, category: 'API::NpmPackages', project: project, user: current_user, namespace: project.namespace)
- created_package
+ authorize_create_package!(project)
+
+ created_package = ::Packages::Npm::CreatePackageService
+ .new(project, current_user, params.merge(build: current_authenticated_job)).execute
+
+ if created_package[:status] == :error
+ render_api_error!(created_package[:message], created_package[:http_status])
+ else
+ track_package_event('push_package', :npm, category: 'API::NpmPackages', project: project, namespace: project.namespace)
+ created_package
+ end
end
end
diff --git a/lib/api/nuget_project_packages.rb b/lib/api/nuget_project_packages.rb
index 8e974cb9cbe..cd16aaf6b5f 100644
--- a/lib/api/nuget_project_packages.rb
+++ b/lib/api/nuget_project_packages.rb
@@ -125,7 +125,6 @@ module API
'push_package',
:nuget,
category: 'API::NugetPackages',
- user: current_user,
project: package.project,
namespace: package.project.namespace
)
@@ -171,7 +170,6 @@ module API
'push_symbol_package',
:nuget,
category: 'API::NugetPackages',
- user: current_user,
project: package.project,
namespace: package.project.namespace
)
diff --git a/lib/api/package_files.rb b/lib/api/package_files.rb
index bb9f96cdbb1..7ff49f326d9 100644
--- a/lib/api/package_files.rb
+++ b/lib/api/package_files.rb
@@ -29,6 +29,7 @@ module API
params do
use :pagination
end
+ route_setting :authentication, job_token_allowed: true
get ':id/packages/:package_id/package_files' do
package = ::Packages::PackageFinder
.new(user_project, params[:package_id]).execute
@@ -51,6 +52,7 @@ module API
params do
requires :package_file_id, type: Integer, desc: 'ID of a package file'
end
+ route_setting :authentication, job_token_allowed: true
delete ':id/packages/:package_id/package_files/:package_file_id' do
authorize_destroy_package!(user_project)
diff --git a/lib/api/personal_access_tokens.rb b/lib/api/personal_access_tokens.rb
index 66930ecd797..9d234ca0593 100644
--- a/lib/api/personal_access_tokens.rb
+++ b/lib/api/personal_access_tokens.rb
@@ -4,7 +4,7 @@ module API
class PersonalAccessTokens < ::API::Base
include ::API::PaginationParams
- feature_category :authentication_and_authorization
+ feature_category :system_access
before do
authenticate!
@@ -68,6 +68,30 @@ module API
end
end
+ desc 'Rotate personal access token' do
+ detail 'Roates a personal access token.'
+ success Entities::PersonalAccessTokenWithToken
+ end
+ post ':id/rotate' do
+ token = PersonalAccessToken.find_by_id(params[:id])
+
+ if Ability.allowed?(current_user, :manage_user_personal_access_token, token&.user)
+ response = ::PersonalAccessTokens::RotateService.new(current_user, token).execute
+
+ if response.success?
+ status :ok
+
+ new_token = response.payload[:personal_access_token]
+ present new_token, with: Entities::PersonalAccessTokenWithToken
+ else
+ bad_request!(response.message)
+ end
+ else
+ # Only admins should be informed if the token doesn't exist
+ current_user.can_admin_all_resources? ? not_found! : unauthorized!
+ end
+ end
+
desc 'Revoke a personal access token' do
detail 'Revoke a personal access token by using the ID of the personal access token.'
success code: 204
diff --git a/lib/api/personal_access_tokens/self_information.rb b/lib/api/personal_access_tokens/self_information.rb
index 5735fe49f33..4f17ca955ac 100644
--- a/lib/api/personal_access_tokens/self_information.rb
+++ b/lib/api/personal_access_tokens/self_information.rb
@@ -5,7 +5,7 @@ module API
class SelfInformation < ::API::Base
include APIGuard
- feature_category :authentication_and_authorization
+ feature_category :system_access
helpers ::API::Helpers::PersonalAccessTokensHelpers
diff --git a/lib/api/project_clusters.rb b/lib/api/project_clusters.rb
index 21f1ee69613..8e5b089434a 100644
--- a/lib/api/project_clusters.rb
+++ b/lib/api/project_clusters.rb
@@ -9,7 +9,7 @@ module API
ensure_feature_enabled!
end
- feature_category :kubernetes_management
+ feature_category :deployment_management
urgency :low
params do
diff --git a/lib/api/project_container_repositories.rb b/lib/api/project_container_repositories.rb
index 0f4f1dc7fa6..3e7e95c2c0d 100644
--- a/lib/api/project_container_repositories.rb
+++ b/lib/api/project_container_repositories.rb
@@ -40,7 +40,7 @@ module API
user: current_user, subject: user_project
).execute
- track_package_event('list_repositories', :container, user: current_user, project: user_project, namespace: user_project.namespace)
+ track_package_event('list_repositories', :container, project: user_project, namespace: user_project.namespace)
present paginate(repositories), with: Entities::ContainerRegistry::Repository, tags: params[:tags], tags_count: params[:tags_count]
end
@@ -62,7 +62,7 @@ module API
authorize_admin_container_image!
repository.delete_scheduled!
- track_package_event('delete_repository', :container, user: current_user, project: user_project, namespace: user_project.namespace)
+ track_package_event('delete_repository', :container, project: user_project, namespace: user_project.namespace)
status :accepted
end
@@ -85,7 +85,7 @@ module API
authorize_read_container_image!
tags = Kaminari.paginate_array(repository.tags)
- track_package_event('list_tags', :container, user: current_user, project: user_project, namespace: user_project.namespace)
+ track_package_event('list_tags', :container, project: user_project, namespace: user_project.namespace)
present paginate(tags), with: Entities::ContainerRegistry::Tag
end
@@ -121,7 +121,7 @@ module API
declared_params.except(:repository_id))
# rubocop:enable CodeReuse/Worker
- track_package_event('delete_tag_bulk', :container, user: current_user, project: user_project, namespace: user_project.namespace)
+ track_package_event('delete_tag_bulk', :container, project: user_project, namespace: user_project.namespace)
status :accepted
end
@@ -169,7 +169,7 @@ module API
.execute(repository)
if result[:status] == :success
- track_package_event('delete_tag', :container, user: current_user, project: user_project, namespace: user_project.namespace)
+ track_package_event('delete_tag', :container, project: user_project, namespace: user_project.namespace)
status :ok
else
diff --git a/lib/api/project_import.rb b/lib/api/project_import.rb
index 02f0d9a2a70..6639b3ec346 100644
--- a/lib/api/project_import.rb
+++ b/lib/api/project_import.rb
@@ -33,16 +33,14 @@ module API
end
end
- before do
- forbidden! unless Gitlab::CurrentSettings.import_sources.include?('gitlab_project')
- end
-
resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
desc 'Workhorse authorize the project import upload' do
detail 'This feature was introduced in GitLab 12.9'
tags ['project_import']
end
post 'import/authorize' do
+ forbidden! unless Gitlab::CurrentSettings.import_sources.include?('gitlab_project')
+
require_gitlab_workhorse!
status 200
@@ -90,6 +88,8 @@ module API
consumes ['multipart/form-data']
end
post 'import' do
+ forbidden! unless Gitlab::CurrentSettings.import_sources.include?('gitlab_project')
+
require_gitlab_workhorse!
check_rate_limit! :project_import, scope: [current_user, :project_import]
@@ -164,6 +164,8 @@ module API
]
end
post 'remote-import' do
+ forbidden! unless Gitlab::CurrentSettings.import_sources.include?('gitlab_project')
+
check_rate_limit! :project_import, scope: [current_user, :project_import]
response = ::Import::GitlabProjects::CreateProjectService.new(
@@ -217,7 +219,7 @@ module API
]
end
post 'remote-import-s3' do
- not_found! unless ::Feature.enabled?(:import_project_from_remote_file_s3)
+ forbidden! unless Gitlab::CurrentSettings.import_sources.include?('gitlab_project')
check_rate_limit! :project_import, scope: [current_user, :project_import]
diff --git a/lib/api/project_job_token_scope.rb b/lib/api/project_job_token_scope.rb
new file mode 100644
index 00000000000..7fd288491ef
--- /dev/null
+++ b/lib/api/project_job_token_scope.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module API
+ class ProjectJobTokenScope < ::API::Base
+ before { authenticate! }
+
+ feature_category :secrets_management
+ urgency :low
+
+ resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
+ desc 'Fetch CI_JOB_TOKEN access settings.' do
+ failure [
+ { code: 401, message: 'Unauthorized' },
+ { code: 403, message: 'Forbidden' },
+ { code: 404, message: 'Not found' }
+ ]
+ success code: 200, model: Entities::ProjectJobTokenScope
+ tags %w[projects_job_token_scope]
+ end
+ get ':id/job_token_scope' do
+ authorize_admin_project
+
+ present user_project, with: Entities::ProjectJobTokenScope
+ end
+ end
+ end
+end
diff --git a/lib/api/project_templates.rb b/lib/api/project_templates.rb
index 8ec67988e39..2360a7e6b2a 100644
--- a/lib/api/project_templates.rb
+++ b/lib/api/project_templates.rb
@@ -32,6 +32,8 @@ module API
use :pagination
end
get ':id/templates/:type' do
+ bad_request! if params[:type] == 'metrics_dashboard_ymls' && Feature.enabled?(:remove_monitor_metrics)
+
templates = TemplateFinder.all_template_names(user_project, params[:type]).values.flatten
present paginate(::Kaminari.paginate_array(templates)), with: Entities::TemplatesList
@@ -60,6 +62,8 @@ module API
end
get ':id/templates/:type/:name', requirements: TEMPLATE_NAMES_ENDPOINT_REQUIREMENTS do
+ bad_request! if params[:type] == 'metrics_dashboard_ymls' && Feature.enabled?(:remove_monitor_metrics)
+
begin
template = TemplateFinder.build(
params[:type],
diff --git a/lib/api/projects.rb b/lib/api/projects.rb
index 6eea56ea117..d6863e4eba4 100644
--- a/lib/api/projects.rb
+++ b/lib/api/projects.rb
@@ -40,6 +40,23 @@ module API
attrs.delete(:repository_storage) unless can?(current_user, :use_project_statistics_filters)
end
+ def validate_updated_at_order_and_filter!
+ return unless filter_by_updated_at? && provided_order_is_not_updated_at?
+
+ # This is necessary as not pairing this filter and ordering will produce an inneficient query
+ bad_request!('`updated_at` filter and `updated_at` sorting must be paired')
+ end
+
+ def provided_order_is_not_updated_at?
+ order_by_param = declared_params[:order_by]
+
+ order_by_param.present? && order_by_param.to_s != 'updated_at'
+ end
+
+ def filter_by_updated_at?
+ declared_params[:updated_before].present? || declared_params[:updated_after].present?
+ end
+
def verify_statistics_order_by_projects!
return unless Helpers::ProjectsHelpers::STATISTICS_SORT_PARAMS.include?(params[:order_by])
@@ -80,9 +97,8 @@ module API
# Temporarily introduced for upload API: https://gitlab.com/gitlab-org/gitlab/-/issues/325788
def project_attachment_size(user_project)
return PROJECT_ATTACHMENT_SIZE_EXEMPT if exempt_from_global_attachment_size?(user_project)
- return user_project.max_attachment_size if Feature.enabled?(:enforce_max_attachment_size_upload_api, user_project)
- PROJECT_ATTACHMENT_SIZE_EXEMPT
+ user_project.max_attachment_size
end
# This is to help determine which projects to use in https://gitlab.com/gitlab-org/gitlab/-/issues/325788
@@ -94,6 +110,10 @@ module API
Gitlab::AppLogger.info({ message: "File exceeds maximum size", file_bytes: file.size, project_id: user_project.id, project_path: user_project.full_path, upload_allowed: allowed })
end
end
+
+ def validate_projects_api_rate_limit_for_unauthenticated_users!
+ check_rate_limit!(:projects_api_rate_limit_unauthenticated, scope: [ip_address]) if current_user.blank?
+ end
end
helpers do
@@ -139,6 +159,8 @@ module API
optional :repository_storage, type: String, desc: 'Which storage shard the repository is on. Available only to admins'
optional :topic, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, desc: 'Comma-separated list of topics. Limit results to projects having all topics'
optional :topic_id, type: Integer, desc: 'Limit results to projects with the assigned topic given by the topic ID'
+ optional :updated_before, type: DateTime, desc: 'Return projects updated before the specified datetime. Format: ISO 8601 YYYY-MM-DDTHH:MM:SSZ'
+ optional :updated_after, type: DateTime, desc: 'Return projects updated after the specified datetime. Format: ISO 8601 YYYY-MM-DDTHH:MM:SSZ'
use :optional_filter_params_ee
end
@@ -256,6 +278,9 @@ module API
desc 'Get a list of visible projects for authenticated user' do
success code: 200, model: Entities::BasicProjectDetails
+ failure [
+ { code: 400, message: 'Bad request' }
+ ]
tags %w[projects]
is_array true
end
@@ -266,6 +291,9 @@ module API
end
# TODO: Set higher urgency https://gitlab.com/gitlab-org/gitlab/-/issues/211495
get feature_category: :projects, urgency: :low do
+ validate_projects_api_rate_limit_for_unauthenticated_users!
+ validate_updated_at_order_and_filter!
+
present_projects load_projects
end
@@ -701,7 +729,7 @@ module API
requires :group_access, type: Integer, values: Gitlab::Access.values, as: :link_group_access, desc: 'The group access level'
optional :expires_at, type: Date, desc: 'Share expiration date'
end
- post ":id/share", feature_category: :authentication_and_authorization do
+ post ":id/share", feature_category: :projects do
authorize! :admin_project, user_project
shared_with_group = Group.find_by_id(params[:group_id])
@@ -731,7 +759,7 @@ module API
requires :group_id, type: Integer, desc: 'The ID of the group'
end
# rubocop: disable CodeReuse/ActiveRecord
- delete ":id/share/:group_id", feature_category: :authentication_and_authorization do
+ delete ":id/share/:group_id", feature_category: :projects do
authorize! :admin_project, user_project
link = user_project.project_group_links.find_by(group_id: params[:group_id])
@@ -822,7 +850,7 @@ module API
optional :skip_users, type: Array[Integer], coerce_with: ::API::Validations::Types::CommaSeparatedToIntegerArray.coerce, desc: 'Filter out users with the specified IDs'
use :pagination
end
- get ':id/users', urgency: :low, feature_category: :authentication_and_authorization do
+ get ':id/users', urgency: :low, feature_category: :system_access do
users = DeclarativePolicy.subject_scope { user_project.team.users }
users = users.search(params[:search]) if params[:search].present?
users = users.where_not_in(params[:skip_users]) if params[:skip_users].present?
diff --git a/lib/api/protected_branches.rb b/lib/api/protected_branches.rb
index 786045684b8..3d9abe23638 100644
--- a/lib/api/protected_branches.rb
+++ b/lib/api/protected_branches.rb
@@ -6,8 +6,6 @@ module API
BRANCH_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge(name: API::NO_SLASH_URL_PART_REGEX)
- before { authorize_admin_project }
-
feature_category :source_code_management
helpers Helpers::ProtectedBranchesHelpers
@@ -33,6 +31,8 @@ module API
end
# rubocop: disable CodeReuse/ActiveRecord
get ':id/protected_branches' do
+ authorize_read_code!
+
protected_branches =
ProtectedBranchesFinder
.new(user_project, params)
@@ -55,6 +55,8 @@ module API
end
# rubocop: disable CodeReuse/ActiveRecord
get ':id/protected_branches/:name', requirements: BRANCH_ENDPOINT_REQUIREMENTS do
+ authorize_read_code!
+
protected_branch = user_project.protected_branches.find_by!(name: params[:name])
present protected_branch, with: Entities::ProtectedBranch, project: user_project
@@ -86,6 +88,8 @@ module API
end
# rubocop: disable CodeReuse/ActiveRecord
post ':id/protected_branches' do
+ authorize_admin_project
+
protected_branch = user_project.protected_branches.find_by(name: params[:name])
if protected_branch
@@ -109,18 +113,22 @@ module API
failure [
{ code: 422, message: 'Push access levels access level has already been taken' },
{ code: 404, message: '404 Project Not Found' },
- { code: 401, message: '401 Unauthorized' }
+ { code: 401, message: '401 Unauthorized' },
+ { code: 400, message: '400 Bad request' }
]
end
params do
requires :name, type: String, desc: 'The name of the branch', documentation: { example: 'main' }
optional :allow_force_push, type: Boolean,
- desc: 'Allow force push for all users with push access.'
+ desc: 'Allow force push for all users with push access.',
+ allow_blank: false
use :optional_params_ee
end
# rubocop: disable CodeReuse/ActiveRecord
patch ':id/protected_branches/:name', requirements: BRANCH_ENDPOINT_REQUIREMENTS do
+ authorize_admin_project
+
protected_branch = user_project.protected_branches.find_by!(name: params[:name])
declared_params = declared_params(include_missing: false)
@@ -148,6 +156,8 @@ module API
end
# rubocop: disable CodeReuse/ActiveRecord
delete ':id/protected_branches/:name', requirements: BRANCH_ENDPOINT_REQUIREMENTS, urgency: :low do
+ authorize_admin_project
+
protected_branch = user_project.protected_branches.find_by!(name: params[:name])
destroy_conditionally!(protected_branch) do
diff --git a/lib/api/pypi_packages.rb b/lib/api/pypi_packages.rb
index 8c4203d8819..027a11738d3 100644
--- a/lib/api/pypi_packages.rb
+++ b/lib/api/pypi_packages.rb
@@ -288,7 +288,7 @@ module API
authorize_upload!(project)
bad_request!('File is too large') if project.actual_limits.exceeded?(:pypi_max_file_size, params[:content].size)
- track_package_event('push_package', :pypi, project: project, user: current_user, namespace: project.namespace)
+ track_package_event('push_package', :pypi, project: project, namespace: project.namespace)
validate_fips! if Gitlab::FIPS.enabled?
diff --git a/lib/api/release/links.rb b/lib/api/release/links.rb
index b21bcb4a903..311fcf9aba1 100644
--- a/lib/api/release/links.rb
+++ b/lib/api/release/links.rb
@@ -65,14 +65,16 @@ module API
end
route_setting :authentication, job_token_allowed: true
post 'links' do
- authorize! :create_release, release
-
- new_link = release.links.create(declared_params(include_missing: false))
-
- if new_link.persisted?
- present new_link, with: Entities::Releases::Link
+ result = ::Releases::Links::CreateService
+ .new(release, current_user, declared_params(include_missing: false))
+ .execute
+
+ if result.success?
+ present result.payload[:link], with: Entities::Releases::Link
+ elsif result.reason == ::Releases::Links::REASON_FORBIDDEN
+ forbidden!
else
- render_api_error!(new_link.errors.messages, 400)
+ render_api_error!(result.message, 400)
end
end
@@ -119,12 +121,16 @@ module API
end
route_setting :authentication, job_token_allowed: true
put do
- authorize! :update_release, release
-
- if link.update(declared_params(include_missing: false))
- present link, with: Entities::Releases::Link
+ result = ::Releases::Links::UpdateService
+ .new(release, current_user, declared_params(include_missing: false))
+ .execute(link)
+
+ if result.success?
+ present result.payload[:link], with: Entities::Releases::Link
+ elsif result.reason == ::Releases::Links::REASON_FORBIDDEN
+ forbidden!
else
- render_api_error!(link.errors.messages, 400)
+ render_api_error!(result.message, 400)
end
end
@@ -139,12 +145,16 @@ module API
end
route_setting :authentication, job_token_allowed: true
delete do
- authorize! :destroy_release, release
-
- if link.destroy
- present link, with: Entities::Releases::Link
+ result = ::Releases::Links::DestroyService
+ .new(release, current_user)
+ .execute(link)
+
+ if result.success?
+ present result.payload[:link], with: Entities::Releases::Link
+ elsif result.reason == ::Releases::Links::REASON_FORBIDDEN
+ forbidden!
else
- render_api_error!(link.errors.messages, 400)
+ render_api_error!(result.message, 400)
end
end
end
diff --git a/lib/api/releases.rb b/lib/api/releases.rb
index ebf1c03e86b..0b31a3e0309 100644
--- a/lib/api/releases.rb
+++ b/lib/api/releases.rb
@@ -387,10 +387,6 @@ module API
authorize! :download_code, user_project
end
- def authorize_read_code!
- authorize! :read_code, user_project
- end
-
def authorize_create_evidence!
# extended in EE
end
diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb
index 6f8d34ea387..295d1d5ab16 100644
--- a/lib/api/repositories.rb
+++ b/lib/api/repositories.rb
@@ -41,7 +41,7 @@ module API
end
end
- before { authorize! :read_code, user_project }
+ before { authorize_read_code! }
feature_category :source_code_management
@@ -63,7 +63,7 @@ module API
end
def assign_blob_vars!(limit:)
- authorize! :read_code, user_project
+ authorize_read_code!
@repo = user_project.repository
diff --git a/lib/api/resource_access_tokens.rb b/lib/api/resource_access_tokens.rb
index 754dfadb5fc..b98ed5ec9ff 100644
--- a/lib/api/resource_access_tokens.rb
+++ b/lib/api/resource_access_tokens.rb
@@ -8,7 +8,7 @@ module API
before { authenticate! }
- feature_category :authentication_and_authorization
+ feature_category :system_access
%w[project group].each do |source_type|
resource source_type.pluralize, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
@@ -119,6 +119,38 @@ module API
bad_request!(token_response.message)
end
end
+
+ desc 'Rotate a resource access token' do
+ detail 'This feature was introduced in GitLab 16.0.'
+ tags ["#{source_type}_access_tokens"]
+ success Entities::ResourceAccessTokenWithToken
+ end
+ params do
+ requires :id, type: String, desc: "The #{source_type} ID"
+ requires :token_id, type: String, desc: "The ID of the token"
+ end
+ post ':id/access_tokens/:token_id/rotate' do
+ resource = find_source(source_type, params[:id])
+
+ resource_accessible = Ability.allowed?(current_user, :manage_resource_access_tokens, resource)
+ token = find_token(resource, params[:token_id]) if resource_accessible
+
+ if token
+ response = ::PersonalAccessTokens::RotateService.new(current_user, token).execute
+
+ if response.success?
+ status :ok
+
+ new_token = response.payload[:personal_access_token]
+ present new_token, with: Entities::ResourceAccessTokenWithToken, resource: resource
+ else
+ bad_request!(response.message)
+ end
+ else
+ # Only admins should be informed if the token doesn't exist
+ current_user.can_admin_all_resources? ? not_found! : unauthorized!
+ end
+ end
end
end
diff --git a/lib/api/rpm_project_packages.rb b/lib/api/rpm_project_packages.rb
index f02d288982a..7db2815bb98 100644
--- a/lib/api/rpm_project_packages.rb
+++ b/lib/api/rpm_project_packages.rb
@@ -97,7 +97,6 @@ module API
track_package_event(
'push_package',
:rpm,
- user: current_user,
category: self.class.name,
project: authorized_user_project,
namespace: authorized_user_project.namespace
diff --git a/lib/api/rubygem_packages.rb b/lib/api/rubygem_packages.rb
index 896d8fcc727..b7bc065eee8 100644
--- a/lib/api/rubygem_packages.rb
+++ b/lib/api/rubygem_packages.rb
@@ -138,7 +138,7 @@ module API
authorize_upload!(project)
bad_request!('File is too large') if project.actual_limits.exceeded?(:rubygems_max_file_size, params[:file].size)
- track_package_event('push_package', :rubygems, user: current_user, project: project, namespace: project.namespace)
+ track_package_event('push_package', :rubygems, project: project, namespace: project.namespace)
package_file = nil
diff --git a/lib/api/search.rb b/lib/api/search.rb
index 2204437f2ec..954c3cd9f9e 100644
--- a/lib/api/search.rb
+++ b/lib/api/search.rb
@@ -68,6 +68,9 @@ module API
@results = search_service.search_objects(preload_method)
end
+ search_results = search_service.search_results
+ bad_request!(search_results.error) if search_results.respond_to?(:failed?) && search_results.failed?
+
set_global_search_log_information(additional_params)
Gitlab::Metrics::GlobalSearchSlis.record_apdex(
diff --git a/lib/api/settings.rb b/lib/api/settings.rb
index 8efb848eb57..7d6e2ee4d4c 100644
--- a/lib/api/settings.rb
+++ b/lib/api/settings.rb
@@ -97,7 +97,7 @@ module API
end
optional :html_emails_enabled, type: Boolean, desc: 'By default GitLab sends emails in HTML and plain text formats so mail clients can choose what format to use. Disable this option if you only want to send emails in plain text format.'
optional :import_sources, type: Array[String], coerce_with: Validations::Types::CommaSeparatedToArray.coerce,
- values: %w[github bitbucket bitbucket_server gitlab fogbugz git gitlab_project gitea manifest phabricator],
+ values: %w[github bitbucket bitbucket_server fogbugz git gitlab_project gitea manifest],
desc: 'Enabled sources for code import during project creation. OmniAuth must be configured for GitHub, Bitbucket, and GitLab.com'
optional :in_product_marketing_emails_enabled, type: Boolean, desc: 'By default, in-product marketing emails are enabled. To disable these emails, disable this option.'
optional :invisible_captcha_enabled, type: Boolean, desc: 'Enable Invisible Captcha spam detection during signup.'
@@ -152,6 +152,7 @@ module API
given shared_runners_enabled: ->(val) { val } do
requires :shared_runners_text, type: String, desc: 'Shared runners text '
end
+ optional :valid_runner_registrars, type: Array[String], desc: 'List of types which are allowed to register a GitLab runner'
optional :sign_in_text, type: String, desc: 'The sign in text of the GitLab application'
optional :signin_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled for the web interface' # support legacy names, can be removed in v5
optional :signup_enabled, type: Boolean, desc: 'Flag indicating if sign up is enabled'
@@ -195,6 +196,8 @@ module API
optional :jira_connect_proxy_url, type: String, desc: "URL of the GitLab instance that should be used as a proxy for the GitLab for Jira Cloud app"
optional :bulk_import_enabled, type: Boolean, desc: 'Enable migrating GitLab groups and projects by direct transfer'
optional :allow_runner_registration_token, type: Boolean, desc: 'Allow registering runners using a registration token'
+ optional :ci_max_includes, type: Integer, desc: 'Maximum number of includes per pipeline'
+ optional :security_policy_global_group_approvers_enabled, type: Boolean, desc: 'Query scan result policy approval groups globally'
Gitlab::SSHPublicKey.supported_types.each do |type|
optional :"#{type}_key_restriction",
diff --git a/lib/api/subscriptions.rb b/lib/api/subscriptions.rb
index 52e5ab30d06..83ad49436bc 100644
--- a/lib/api/subscriptions.rb
+++ b/lib/api/subscriptions.rb
@@ -87,8 +87,6 @@ module API
case source_type
when 'project'
user_project
- else
- nil
end
end
end
diff --git a/lib/api/tags.rb b/lib/api/tags.rb
index f918fb997bf..42b63af59e0 100644
--- a/lib/api/tags.rb
+++ b/lib/api/tags.rb
@@ -7,7 +7,7 @@ module API
TAG_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge(tag_name: API::NO_SLASH_URL_PART_REGEX)
before do
- authorize! :read_code, user_project
+ authorize_read_code!
not_found! unless user_project.repo_exists?
end
diff --git a/lib/api/templates.rb b/lib/api/templates.rb
index a80ef514943..583b6e1fed4 100644
--- a/lib/api/templates.rb
+++ b/lib/api/templates.rb
@@ -12,7 +12,7 @@ module API
},
gitlab_ci_ymls: {
gitlab_version: 8.9,
- feature_category: :pipeline_authoring,
+ feature_category: :pipeline_composition,
file_type: 'GitLab CI/CD YAML'
},
dockerfiles: {
diff --git a/lib/api/terraform/modules/v1/packages.rb b/lib/api/terraform/modules/v1/packages.rb
index 5624784228e..8f264097867 100644
--- a/lib/api/terraform/modules/v1/packages.rb
+++ b/lib/api/terraform/modules/v1/packages.rb
@@ -200,7 +200,7 @@ module API
tags %w[terraform_registry]
end
get do
- track_package_event('pull_package', :terraform_module, project: package.project, namespace: module_namespace, user: current_user)
+ track_package_event('pull_package', :terraform_module, project: package.project, namespace: module_namespace)
present_carrierwave_file!(package_file.file)
end
@@ -292,7 +292,7 @@ module API
render_api_error!(result[:message], result[:http_status]) if result[:status] == :error
- track_package_event('push_package', :terraform_module, project: authorized_user_project, user: current_user, namespace: authorized_user_project.namespace)
+ track_package_event('push_package', :terraform_module, project: authorized_user_project, namespace: authorized_user_project.namespace)
created!
rescue ObjectStorage::RemoteStoreError => e
diff --git a/lib/api/terraform/state.rb b/lib/api/terraform/state.rb
index bdc9f975970..8017a195f28 100644
--- a/lib/api/terraform/state.rb
+++ b/lib/api/terraform/state.rb
@@ -28,18 +28,16 @@ module API
increment_unique_values('p_terraform_state_api_unique_users', current_user.id)
- if Feature.enabled?(:route_hll_to_snowplow_phase2, user_project&.namespace)
- Gitlab::Tracking.event(
- 'API::Terraform::State',
- 'terraform_state_api_request',
- namespace: user_project&.namespace,
- user: current_user,
- project: user_project,
- label: 'redis_hll_counters.terraform.p_terraform_state_api_unique_users_monthly',
- context: [Gitlab::Tracking::ServicePingContext.new(data_source: :redis_hll,
- event: 'p_terraform_state_api_unique_users').to_context]
- )
- end
+ Gitlab::Tracking.event(
+ 'API::Terraform::State',
+ 'terraform_state_api_request',
+ namespace: user_project&.namespace,
+ user: current_user,
+ project: user_project,
+ label: 'redis_hll_counters.terraform.p_terraform_state_api_unique_users_monthly',
+ context: [Gitlab::Tracking::ServicePingContext.new(data_source: :redis_hll,
+ event: 'p_terraform_state_api_unique_users').to_context]
+ )
end
params do
@@ -57,17 +55,6 @@ module API
def remote_state_handler
::Terraform::RemoteStateHandler.new(user_project, current_user, name: params[:name], lock_id: params[:ID])
end
-
- def not_found_for_dots?
- Feature.disabled?(:allow_dots_on_tf_state_names) && params[:name].include?(".")
- end
-
- # Change the state name to behave like before, https://gitlab.com/gitlab-org/gitlab/-/merge_requests/105674
- # has been introduced. This behavior can be controlled via `allow_dots_on_tf_state_names` FF.
- # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/106861
- def legacy_state_name!
- params[:name] = params[:name].split('.').first
- end
end
desc 'Get a Terraform state by its name' do
@@ -85,8 +72,6 @@ module API
end
route_setting :authentication, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth
get do
- legacy_state_name! if not_found_for_dots?
-
remote_state_handler.find_with_lock do |state|
no_content! unless state.latest_file && state.latest_file.exists?
@@ -111,7 +96,6 @@ module API
route_setting :authentication, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth
post do
authorize! :admin_terraform_state, user_project
- legacy_state_name! if not_found_for_dots?
data = request.body.read
no_content! if data.empty?
@@ -140,7 +124,6 @@ module API
route_setting :authentication, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth
delete do
authorize! :admin_terraform_state, user_project
- legacy_state_name! if not_found_for_dots?
remote_state_handler.find_with_lock do |state|
::Terraform::States::TriggerDestroyService.new(state, current_user: current_user).execute
@@ -172,8 +155,6 @@ module API
requires :Path, type: String, desc: 'Terraform path'
end
post '/lock' do
- not_found! if not_found_for_dots?
-
authorize! :admin_terraform_state, user_project
status_code = :ok
@@ -217,8 +198,6 @@ module API
optional :ID, type: String, limit: 255, desc: 'Terraform state lock ID'
end
delete '/lock' do
- not_found! if not_found_for_dots?
-
authorize! :admin_terraform_state, user_project
remote_state_handler.unlock!
diff --git a/lib/api/todos.rb b/lib/api/todos.rb
index 57745ee8802..7e6b21d6121 100644
--- a/lib/api/todos.rb
+++ b/lib/api/todos.rb
@@ -88,6 +88,8 @@ module API
use :pagination, :todo_filters
end
get do
+ Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/408576')
+
todos = paginate(find_todos.with_entity_associations)
todos = ::Todos::AllowedTargetFilterService.new(todos, current_user).execute
options = { with: Entities::Todo, current_user: current_user }
diff --git a/lib/api/unleash.rb b/lib/api/unleash.rb
index b2133dc743a..ae84c6c7010 100644
--- a/lib/api/unleash.rb
+++ b/lib/api/unleash.rb
@@ -33,13 +33,11 @@ module API
present_feature_flags
end
- # We decrease the urgency of this endpoint until the maxmemory issue of redis-cache has been resolved.
- # See https://gitlab.com/gitlab-org/gitlab/-/issues/365575#note_1033611872 for more information.
desc 'Get a list of features' do
is_array true
tags unleash_tags
end
- get 'client/features', urgency: :low do
+ get 'client/features', urgency: :medium do
present_feature_flags
end
diff --git a/lib/api/users.rb b/lib/api/users.rb
index cc7eb63798a..3d9af536c3c 100644
--- a/lib/api/users.rb
+++ b/lib/api/users.rb
@@ -80,31 +80,6 @@ module API
end
end
- resources ':id/associations_count' do
- helpers do
- def present_entity(result)
- present result,
- with: ::API::Entities::UserAssociationsCount
- end
- end
-
- desc "Returns a list of a specified user's count of projects, groups, issues and merge requests."
- params do
- requires :id,
- type: Integer,
- desc: 'ID of the user to query.'
- end
- get do
- authenticate!
-
- user = find_user_by_id(params)
- forbidden! unless can?(current_user, :get_user_associations_count, user)
- not_found!('User') unless user
-
- present_entity(user)
- end
- end
-
desc 'Get the list of users' do
success Entities::UserBasic
end
@@ -156,7 +131,7 @@ module API
entity = current_user&.can_read_all_resources? ? Entities::UserWithAdmin : Entities::UserBasic
if entity == Entities::UserWithAdmin
- users = users.preload(:identities, :u2f_registrations, :webauthn_registrations, :namespace, :followers, :followees, :user_preference)
+ users = users.preload(:identities, :webauthn_registrations, :namespace, :followers, :followees, :user_preference)
end
users, options = with_custom_attributes(users, { with: entity, current_user: current_user })
@@ -220,12 +195,13 @@ module API
not_found!('User') unless user
followee = current_user.follow(user)
+
+ not_modified! unless followee
+
if followee&.errors&.any?
render_api_error!(followee.errors.full_messages.join(', '), 400)
elsif followee&.persisted?
present user, with: Entities::UserBasic
- else
- not_modified!
end
end
@@ -381,7 +357,7 @@ module API
params do
requires :id, type: Integer, desc: 'The ID of the user'
end
- patch ":id/disable_two_factor", feature_category: :authentication_and_authorization do
+ patch ":id/disable_two_factor", feature_category: :system_access do
authenticated_as_admin!
user = User.find_by_id(params[:id])
@@ -407,7 +383,7 @@ module API
requires :provider, type: String, desc: 'The external provider'
end
# rubocop: disable CodeReuse/ActiveRecord
- delete ":id/identities/:provider", feature_category: :authentication_and_authorization do
+ delete ":id/identities/:provider", feature_category: :system_access do
authenticated_as_admin!
user = User.find_by(id: params[:id])
@@ -456,7 +432,7 @@ module API
desc: 'Scope of usage for the SSH key'
end
# rubocop: disable CodeReuse/ActiveRecord
- post ":user_id/keys", feature_category: :authentication_and_authorization do
+ post ":user_id/keys", feature_category: :system_access do
authenticated_as_admin!
user = User.find_by(id: params.delete(:user_id))
@@ -479,7 +455,7 @@ module API
requires :user_id, type: String, desc: 'The ID or username of the user'
use :pagination
end
- get ':user_id/keys', requirements: API::USER_REQUIREMENTS, feature_category: :authentication_and_authorization do
+ get ':user_id/keys', requirements: API::USER_REQUIREMENTS, feature_category: :system_access do
user = find_user(params[:user_id])
not_found!('User') unless user && can?(current_user, :read_user, user)
@@ -494,7 +470,7 @@ module API
requires :id, type: Integer, desc: 'The ID of the user'
requires :key_id, type: Integer, desc: 'The ID of the SSH key'
end
- get ':id/keys/:key_id', requirements: API::USER_REQUIREMENTS, feature_category: :authentication_and_authorization do
+ get ':id/keys/:key_id', requirements: API::USER_REQUIREMENTS, feature_category: :system_access do
user = find_user(params[:id])
not_found!('User') unless user && can?(current_user, :read_user, user)
@@ -512,7 +488,7 @@ module API
requires :key_id, type: Integer, desc: 'The ID of the SSH key'
end
# rubocop: disable CodeReuse/ActiveRecord
- delete ':id/keys/:key_id', feature_category: :authentication_and_authorization do
+ delete ':id/keys/:key_id', feature_category: :system_access do
authenticated_as_admin!
user = User.find_by(id: params[:id])
@@ -537,7 +513,7 @@ module API
requires :key, type: String, desc: 'The new GPG key'
end
# rubocop: disable CodeReuse/ActiveRecord
- post ':id/gpg_keys', feature_category: :authentication_and_authorization do
+ post ':id/gpg_keys', feature_category: :system_access do
authenticated_as_admin!
user = User.find_by(id: params.delete(:id))
@@ -562,7 +538,7 @@ module API
use :pagination
end
# rubocop: disable CodeReuse/ActiveRecord
- get ':id/gpg_keys', feature_category: :authentication_and_authorization do
+ get ':id/gpg_keys', feature_category: :system_access do
user = User.find_by(id: params[:id])
not_found!('User') unless user
@@ -579,7 +555,7 @@ module API
requires :key_id, type: Integer, desc: 'The ID of the GPG key'
end
# rubocop: disable CodeReuse/ActiveRecord
- get ':id/gpg_keys/:key_id', feature_category: :authentication_and_authorization do
+ get ':id/gpg_keys/:key_id', feature_category: :system_access do
user = User.find_by(id: params[:id])
not_found!('User') unless user
@@ -598,7 +574,7 @@ module API
requires :key_id, type: Integer, desc: 'The ID of the GPG key'
end
# rubocop: disable CodeReuse/ActiveRecord
- delete ':id/gpg_keys/:key_id', feature_category: :authentication_and_authorization do
+ delete ':id/gpg_keys/:key_id', feature_category: :system_access do
authenticated_as_admin!
user = User.find_by(id: params[:id])
@@ -622,7 +598,7 @@ module API
requires :key_id, type: Integer, desc: 'The ID of the GPG key'
end
# rubocop: disable CodeReuse/ActiveRecord
- post ':id/gpg_keys/:key_id/revoke', feature_category: :authentication_and_authorization do
+ post ':id/gpg_keys/:key_id/revoke', feature_category: :system_access do
authenticated_as_admin!
user = User.find_by(id: params[:id])
@@ -726,7 +702,7 @@ module API
requires :id, type: Integer, desc: 'The ID of the user'
end
# rubocop: disable CodeReuse/ActiveRecord
- post ':id/activate', feature_category: :authentication_and_authorization do
+ post ':id/activate', feature_category: :system_access do
authenticated_as_admin!
user = User.find_by(id: params[:id])
@@ -740,7 +716,7 @@ module API
params do
requires :id, type: Integer, desc: 'The ID of the user'
end
- post ':id/approve', feature_category: :authentication_and_authorization do
+ post ':id/approve', feature_category: :system_access do
user = User.find_by(id: params[:id])
not_found!('User') unless can?(current_user, :read_user, user)
@@ -757,7 +733,7 @@ module API
params do
requires :id, type: Integer, desc: 'The ID of the user'
end
- post ':id/reject', feature_category: :authentication_and_authorization do
+ post ':id/reject', feature_category: :system_access do
user = find_user_by_id(params)
result = ::Users::RejectService.new(current_user).execute(user)
@@ -775,23 +751,18 @@ module API
requires :id, type: Integer, desc: 'The ID of the user'
end
# rubocop: disable CodeReuse/ActiveRecord
- post ':id/deactivate', feature_category: :authentication_and_authorization do
+ post ':id/deactivate', feature_category: :system_access do
authenticated_as_admin!
user = User.find_by(id: params[:id])
not_found!('User') unless user
break if user.deactivated?
- unless user.can_be_deactivated?
- forbidden!('A blocked user cannot be deactivated by the API') if user.blocked?
- forbidden!('An internal user cannot be deactivated by the API') if user.internal?
- forbidden!("The user you are trying to deactivate has been active in the past #{Gitlab::CurrentSettings.deactivate_dormant_users_period} days and cannot be deactivated")
- end
-
- if user.deactivate
+ result = ::Users::DeactivateService.new(current_user, skip_authorization: true).execute(user)
+ if result[:status] == :success
true
else
- render_api_error!(user.errors.full_messages, 400)
+ render_api_error!(result[:message], result[:reason] || :bad_request)
end
end
# rubocop: enable CodeReuse/ActiveRecord
@@ -801,7 +772,7 @@ module API
requires :id, type: Integer, desc: 'The ID of the user'
end
# rubocop: disable CodeReuse/ActiveRecord
- post ':id/block', feature_category: :authentication_and_authorization do
+ post ':id/block', feature_category: :system_access do
authenticated_as_admin!
user = User.find_by(id: params[:id])
not_found!('User') unless user
@@ -828,7 +799,7 @@ module API
requires :id, type: Integer, desc: 'The ID of the user'
end
# rubocop: disable CodeReuse/ActiveRecord
- post ':id/unblock', feature_category: :authentication_and_authorization do
+ post ':id/unblock', feature_category: :system_access do
authenticated_as_admin!
user = User.find_by(id: params[:id])
not_found!('User') unless user
@@ -848,7 +819,7 @@ module API
params do
requires :id, type: Integer, desc: 'The ID of the user'
end
- post ':id/ban', feature_category: :authentication_and_authorization do
+ post ':id/ban', feature_category: :system_access do
authenticated_as_admin!
user = find_user_by_id(params)
@@ -864,7 +835,7 @@ module API
params do
requires :id, type: Integer, desc: 'The ID of the user'
end
- post ':id/unban', feature_category: :authentication_and_authorization do
+ post ':id/unban', feature_category: :system_access do
authenticated_as_admin!
user = find_user_by_id(params)
@@ -902,6 +873,31 @@ module API
present paginate(members), with: Entities::Membership
end
+ resources ':id/associations_count' do
+ helpers do
+ def present_entity(result)
+ present result,
+ with: ::API::Entities::UserAssociationsCount
+ end
+ end
+
+ desc "Returns a list of a specified user's count of projects, groups, issues and merge requests."
+ params do
+ requires :id,
+ type: Integer,
+ desc: 'ID of the user to query.'
+ end
+ get do
+ authenticate!
+
+ user = find_user_by_id(params)
+ forbidden! unless can?(current_user, :get_user_associations_count, user)
+ not_found!('User') unless user
+
+ present_entity(user)
+ end
+ end
+
params do
requires :user_id, type: Integer, desc: 'The ID of the user'
end
@@ -928,7 +924,7 @@ module API
use :pagination
optional :state, type: String, default: 'all', values: %w[all active inactive], desc: 'Filters (all|active|inactive) impersonation_tokens'
end
- get feature_category: :authentication_and_authorization do
+ get feature_category: :system_access do
present paginate(finder(declared_params(include_missing: false)).execute), with: Entities::ImpersonationToken
end
@@ -941,7 +937,7 @@ module API
optional :expires_at, type: Date, desc: 'The expiration date in the format YEAR-MONTH-DAY of the impersonation token'
optional :scopes, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, desc: 'The array of scopes of the impersonation token'
end
- post feature_category: :authentication_and_authorization do
+ post feature_category: :system_access do
impersonation_token = finder.build(declared_params(include_missing: false))
if impersonation_token.save
@@ -958,7 +954,7 @@ module API
params do
requires :impersonation_token_id, type: Integer, desc: 'The ID of the impersonation token'
end
- get ':impersonation_token_id', feature_category: :authentication_and_authorization do
+ get ':impersonation_token_id', feature_category: :system_access do
present find_impersonation_token, with: Entities::ImpersonationToken
end
@@ -968,7 +964,7 @@ module API
params do
requires :impersonation_token_id, type: Integer, desc: 'The ID of the impersonation token'
end
- delete ':impersonation_token_id', feature_category: :authentication_and_authorization do
+ delete ':impersonation_token_id', feature_category: :system_access do
token = find_impersonation_token
destroy_conditionally!(token) do
@@ -996,7 +992,7 @@ module API
desc: 'The array of scopes of the personal access token'
optional :expires_at, type: Date, desc: 'The expiration date in the format YEAR-MONTH-DAY of the personal access token'
end
- post feature_category: :authentication_and_authorization do
+ post feature_category: :system_access do
response = ::PersonalAccessTokens::CreateService.new(
current_user: current_user, target_user: target_user, params: declared_params(include_missing: false)
).execute
@@ -1060,7 +1056,7 @@ module API
params do
use :pagination
end
- get "keys", feature_category: :authentication_and_authorization do
+ get "keys", feature_category: :system_access do
keys = current_user.keys.preload_users
present paginate(keys), with: Entities::SSHKey
@@ -1073,7 +1069,7 @@ module API
requires :key_id, type: Integer, desc: 'The ID of the SSH key'
end
# rubocop: disable CodeReuse/ActiveRecord
- get "keys/:key_id", feature_category: :authentication_and_authorization do
+ get "keys/:key_id", feature_category: :system_access do
key = current_user.keys.find_by(id: params[:key_id])
not_found!('Key') unless key
@@ -1091,7 +1087,7 @@ module API
optional :usage_type, type: String, values: Key.usage_types.keys, default: 'auth_and_signing',
desc: 'Scope of usage for the SSH key'
end
- post "keys", feature_category: :authentication_and_authorization do
+ post "keys", feature_category: :system_access do
key = ::Keys::CreateService.new(current_user, declared_params(include_missing: false)).execute
if key.persisted?
@@ -1108,7 +1104,7 @@ module API
requires :key_id, type: Integer, desc: 'The ID of the SSH key'
end
# rubocop: disable CodeReuse/ActiveRecord
- delete "keys/:key_id", feature_category: :authentication_and_authorization do
+ delete "keys/:key_id", feature_category: :system_access do
key = current_user.keys.find_by(id: params[:key_id])
not_found!('Key') unless key
@@ -1126,7 +1122,7 @@ module API
params do
use :pagination
end
- get 'gpg_keys', feature_category: :authentication_and_authorization do
+ get 'gpg_keys', feature_category: :system_access do
present paginate(current_user.gpg_keys), with: Entities::GpgKey
end
@@ -1138,7 +1134,7 @@ module API
requires :key_id, type: Integer, desc: 'The ID of the GPG key'
end
# rubocop: disable CodeReuse/ActiveRecord
- get 'gpg_keys/:key_id', feature_category: :authentication_and_authorization do
+ get 'gpg_keys/:key_id', feature_category: :system_access do
key = current_user.gpg_keys.find_by(id: params[:key_id])
not_found!('GPG Key') unless key
@@ -1153,7 +1149,7 @@ module API
params do
requires :key, type: String, desc: 'The new GPG key'
end
- post 'gpg_keys', feature_category: :authentication_and_authorization do
+ post 'gpg_keys', feature_category: :system_access do
key = ::GpgKeys::CreateService.new(current_user, declared_params(include_missing: false)).execute
if key.persisted?
@@ -1170,7 +1166,7 @@ module API
requires :key_id, type: Integer, desc: 'The ID of the GPG key'
end
# rubocop: disable CodeReuse/ActiveRecord
- post 'gpg_keys/:key_id/revoke', feature_category: :authentication_and_authorization do
+ post 'gpg_keys/:key_id/revoke', feature_category: :system_access do
key = current_user.gpg_keys.find_by(id: params[:key_id])
not_found!('GPG Key') unless key
@@ -1186,7 +1182,7 @@ module API
requires :key_id, type: Integer, desc: 'The ID of the SSH key'
end
# rubocop: disable CodeReuse/ActiveRecord
- delete 'gpg_keys/:key_id', feature_category: :authentication_and_authorization do
+ delete 'gpg_keys/:key_id', feature_category: :system_access do
key = current_user.gpg_keys.find_by(id: params[:key_id])
not_found!('GPG Key') unless key
@@ -1227,7 +1223,7 @@ module API
attrs = declared_params(include_missing: false)
- service = ::Users::UpsertCreditCardValidationService.new(attrs, user).execute
+ service = ::Users::UpsertCreditCardValidationService.new(attrs).execute
if service.success?
present user.credit_card_validation, with: Entities::UserCreditCardValidations
@@ -1243,7 +1239,8 @@ module API
params do
optional :view_diffs_file_by_file, type: Boolean, desc: 'Flag indicating the user sees only one file diff per page'
optional :show_whitespace_in_diffs, type: Boolean, desc: 'Flag indicating the user sees whitespace changes in diffs'
- at_least_one_of :view_diffs_file_by_file, :show_whitespace_in_diffs
+ optional :pass_user_identities_to_ci_jwt, type: Boolean, desc: 'Flag indicating the user passes their external identities as CI information'
+ at_least_one_of :view_diffs_file_by_file, :show_whitespace_in_diffs, :pass_user_identities_to_ci_jwt
end
put "preferences", feature_category: :user_profile, urgency: :high do
authenticate!
@@ -1365,6 +1362,63 @@ module API
get 'status', feature_category: :user_profile do
present current_user.status || {}, with: Entities::UserStatus
end
+
+ desc 'Create a runner owned by currently authenticated user' do
+ detail 'Create a new runner'
+ success Entities::Ci::RunnerRegistrationDetails
+ failure [[400, 'Bad Request'], [403, 'Forbidden']]
+ tags %w[user runners]
+ end
+ params do
+ requires :runner_type, type: String, values: ::Ci::Runner.runner_types.keys,
+ desc: %q(Specifies the scope of the runner)
+ given runner_type: ->(runner_type) { runner_type == 'group_type' } do
+ requires :group_id, type: Integer,
+ desc: 'The ID of the group that the runner is created in',
+ documentation: { example: 1 }
+ end
+ given runner_type: ->(runner_type) { runner_type == 'project_type' } do
+ requires :project_id, type: Integer,
+ desc: 'The ID of the project that the runner is created in',
+ documentation: { example: 1 }
+ end
+ optional :description, type: String, desc: %q(Description of the runner)
+ optional :maintenance_note, type: String,
+ desc: %q(Free-form maintenance notes for the runner (1024 characters))
+ optional :paused, type: Boolean, desc: 'Specifies if the runner should ignore new jobs (defaults to false)'
+ optional :locked, type: Boolean,
+ desc: 'Specifies if the runner should be locked for the current project (defaults to false)'
+ optional :access_level, type: String, values: ::Ci::Runner.access_levels.keys,
+ desc: 'The access level of the runner'
+ optional :run_untagged, type: Boolean,
+ desc: 'Specifies if the runner should handle untagged jobs (defaults to true)'
+ optional :tag_list, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce,
+ desc: %q(A list of runner tags)
+ optional :maximum_timeout, type: Integer,
+ desc: 'Maximum timeout that limits the amount of time (in seconds) that runners can run jobs'
+ end
+ post 'runners', urgency: :low, feature_category: :runner_fleet do
+ attributes = attributes_for_keys(
+ %i[runner_type group_id project_id description maintenance_note paused locked run_untagged tag_list
+ access_level maximum_timeout]
+ )
+
+ case attributes[:runner_type]
+ when 'group_type'
+ attributes[:scope] = ::Group.find_by_id(attributes.delete(:group_id))
+ when 'project_type'
+ attributes[:scope] = ::Project.find_by_id(attributes.delete(:project_id))
+ end
+
+ result = ::Ci::Runners::CreateRunnerService.new(user: current_user, params: attributes).execute
+ if result.error?
+ message = result.errors.to_sentence
+ forbidden!(message) if result.reason == :forbidden
+ bad_request!(message)
+ end
+
+ present result.payload[:runner], with: Entities::Ci::RunnerRegistrationDetails
+ end
end
end
end
diff --git a/lib/api/v3/github.rb b/lib/api/v3/github.rb
index db71e823b1d..7d8c37cd39b 100644
--- a/lib/api/v3/github.rb
+++ b/lib/api/v3/github.rb
@@ -1,10 +1,11 @@
# frozen_string_literal: true
+# The endpoints by default return `404` in preparation for their removal
+# (also see comment above `#reversible_end_of_life!`).
+# https://gitlab.com/gitlab-org/gitlab/-/issues/362168
+#
# These endpoints partially mimic Github API behavior in order to successfully
# integrate with Jira Development Panel.
-# Endpoints returning an empty list were temporarily added to avoid 404's
-# during Jira's DVCS integration.
-#
module API
module V3
class Github < ::API::Base
@@ -28,6 +29,8 @@ module API
feature_category :integrations
before do
+ reversible_end_of_life!
+
authorize_jira_user_agent!(request)
authenticate!
end
@@ -38,6 +41,17 @@ module API
requires :project, type: String
end
+ # The endpoints in this class have been deprecated since 15.1.
+ #
+ # Due to uncertainty about the impact of a full removal in 16.0, all endpoints return `404`
+ # by default but we allow customers to toggle a flag to reverse this breaking change.
+ # See https://gitlab.com/gitlab-org/gitlab/-/issues/362168#note_1347692683.
+ #
+ # TODO Make the breaking change irreversible https://gitlab.com/gitlab-org/gitlab/-/issues/408148.
+ def reversible_end_of_life!
+ not_found! unless Feature.enabled?(:jira_dvcs_end_of_life_amnesty)
+ end
+
def authorize_jira_user_agent!(request)
not_found! unless Gitlab::Jira::Middleware.jira_dvcs_connector?(request.env)
end
diff --git a/lib/api/validations/validators/bulk_imports.rb b/lib/api/validations/validators/bulk_imports.rb
index 4625f2f39cd..bff3424a0ac 100644
--- a/lib/api/validations/validators/bulk_imports.rb
+++ b/lib/api/validations/validators/bulk_imports.rb
@@ -6,13 +6,25 @@ module API
module BulkImports
class DestinationSlugPath < Grape::Validations::Base
def validate_param!(attr_name, params)
- unless params[attr_name] =~ Gitlab::Regex.group_path_regex # rubocop: disable Style/GuardClause
+ if Feature.disabled?(:restrict_special_characters_in_namespace_path)
+ return if params[attr_name] =~ Gitlab::Regex.group_path_regex
+
raise Grape::Exceptions::Validation.new(
params: [@scope.full_name(attr_name)],
- message: "cannot start with a dash or forward slash, or end with a period or forward slash. " \
+ message: "#{Gitlab::Regex.group_path_regex_message} " \
"It can only contain alphanumeric characters, periods, underscores, and dashes. " \
- "E.g. 'destination_namespace' not 'destination/namespace'"
+ "For example, 'destination_namespace' not 'destination/namespace'"
)
+ else
+ return if params[attr_name] =~ Gitlab::Regex.oci_repository_path_regex
+
+ raise Grape::Exceptions::Validation.new(
+ params: [@scope.full_name(attr_name)],
+ message: "#{Gitlab::Regex.oci_repository_path_regex_message} " \
+ "It can only contain alphanumeric characters, periods, underscores, and dashes. " \
+ "For example, 'destination_namespace' not 'destination/namespace'"
+ )
+
end
end
end
@@ -21,26 +33,24 @@ module API
def validate_param!(attr_name, params)
return if params[attr_name].blank?
- unless params[attr_name] =~ Gitlab::Regex.bulk_import_destination_namespace_path_regex # rubocop: disable Style/GuardClause
- raise Grape::Exceptions::Validation.new(
- params: [@scope.full_name(attr_name)],
- message: "cannot start with a dash or forward slash, or end with a period or forward slash. " \
- "It can only contain alphanumeric characters, periods, underscores, forward slashes " \
- "and dashes. E.g. 'destination_namespace' or 'destination/namespace'"
- )
- end
+ return if params[attr_name] =~ Gitlab::Regex.bulk_import_destination_namespace_path_regex
+
+ raise Grape::Exceptions::Validation.new(
+ params: [@scope.full_name(attr_name)],
+ message: Gitlab::Regex.bulk_import_destination_namespace_path_regex_message
+ )
end
end
class SourceFullPath < Grape::Validations::Base
def validate_param!(attr_name, params)
- unless params[attr_name] =~ Gitlab::Regex.bulk_import_source_full_path_regex # rubocop: disable Style/GuardClause
- raise Grape::Exceptions::Validation.new(
- params: [@scope.full_name(attr_name)],
- message: "must be a relative path and not include protocol, sub-domain, or domain information. " \
- "E.g. 'source/full/path' not 'https://example.com/source/full/path'" \
- )
- end
+ return if params[attr_name] =~ Gitlab::Regex.bulk_import_source_full_path_regex
+
+ raise Grape::Exceptions::Validation.new(
+ params: [@scope.full_name(attr_name)],
+ message: "must be a relative path and not include protocol, sub-domain, or domain information. " \
+ "For example, 'source/full/path' not 'https://example.com/source/full/path'" \
+ )
end
end
end
diff --git a/lib/atlassian/jira_connect/serializers/branch_entity.rb b/lib/atlassian/jira_connect/serializers/branch_entity.rb
index c663575b7a8..682b2d77102 100644
--- a/lib/atlassian/jira_connect/serializers/branch_entity.rb
+++ b/lib/atlassian/jira_connect/serializers/branch_entity.rb
@@ -7,14 +7,13 @@ module Atlassian
expose :id do |branch|
Digest::SHA256.hexdigest(branch.name)
end
- expose :issueKeys do |branch|
- JiraIssueKeyExtractor.new(branch.name).issue_keys
+ expose :issueKeys do |branch, options|
+ JiraIssueKeyExtractors::Branch.new(options[:project], branch.name).issue_keys
end
expose :name
expose :lastCommit, using: JiraConnect::Serializers::CommitEntity do |branch, options|
options[:project].commit(branch.dereferenced_target)
end
-
expose :url do |branch, options|
project_commits_url(options[:project], branch.name)
end
diff --git a/lib/atlassian/jira_connect/serializers/build_entity.rb b/lib/atlassian/jira_connect/serializers/build_entity.rb
index aa864cb268f..b595d0c2a92 100644
--- a/lib/atlassian/jira_connect/serializers/build_entity.rb
+++ b/lib/atlassian/jira_connect/serializers/build_entity.rb
@@ -22,9 +22,10 @@ module Atlassian
expose :references
def issue_keys
- # extract Jira issue keys from either the source branch/ref or the
- # merge request title.
- @issue_keys ||= pipeline.all_merge_requests.flat_map do |mr|
+ commit_message_issue_keys = JiraIssueKeyExtractor.new(pipeline.git_commit_message).issue_keys
+
+ # extract Jira issue keys from either the source branch/ref or the merge request title.
+ @issue_keys ||= commit_message_issue_keys + pipeline.all_merge_requests.flat_map do |mr|
src = "#{mr.source_branch} #{mr.title} #{mr.description}"
JiraIssueKeyExtractor.new(src).issue_keys
end.uniq
diff --git a/lib/atlassian/jira_connect/serializers/commit_entity.rb b/lib/atlassian/jira_connect/serializers/commit_entity.rb
index 12eb1ed15ea..8aa46984643 100644
--- a/lib/atlassian/jira_connect/serializers/commit_entity.rb
+++ b/lib/atlassian/jira_connect/serializers/commit_entity.rb
@@ -22,10 +22,16 @@ module Atlassian
end
expose :author, using: JiraConnect::Serializers::AuthorEntity
expose :fileCount do |commit|
- commit.stats.total
+ # n+1: https://gitlab.com/gitlab-org/gitaly/-/issues/3375
+ Gitlab::GitalyClient.allow_n_plus_1_calls do
+ commit.stats.total
+ end
end
expose :files do |commit, options|
- files = commit.diffs(max_files: 10).diff_files
+ # n+1: https://gitlab.com/gitlab-org/gitaly/-/issues/3374
+ files = Gitlab::GitalyClient.allow_n_plus_1_calls do
+ commit.diffs(max_files: 10).diff_files
+ end
JiraConnect::Serializers::FileEntity.represent files, options.merge(commit: commit)
end
expose :created_at, as: :authorTimestamp
diff --git a/lib/atlassian/jira_issue_key_extractor.rb b/lib/atlassian/jira_issue_key_extractor.rb
index 968e8b0f82e..881ba4544b2 100644
--- a/lib/atlassian/jira_issue_key_extractor.rb
+++ b/lib/atlassian/jira_issue_key_extractor.rb
@@ -6,12 +6,13 @@ module Atlassian
new(...).issue_keys.any?
end
- def initialize(*text)
+ def initialize(*text, custom_regex: nil)
@text = text.join(' ')
+ @match_regex = custom_regex || Gitlab::Regex.jira_issue_key_regex
end
def issue_keys
- @text.scan(Gitlab::Regex.jira_issue_key_regex).uniq
+ @text.scan(@match_regex).flatten.uniq
end
end
end
diff --git a/lib/atlassian/jira_issue_key_extractors/branch.rb b/lib/atlassian/jira_issue_key_extractors/branch.rb
new file mode 100644
index 00000000000..0669cd8ed61
--- /dev/null
+++ b/lib/atlassian/jira_issue_key_extractors/branch.rb
@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Atlassian
+ module JiraIssueKeyExtractors
+ class Branch
+ def self.has_keys?(project, branch_name)
+ new(project, branch_name).issue_keys.any?
+ end
+
+ def initialize(project, branch_name)
+ @project = project
+ @branch_name = branch_name
+ end
+
+ # Extract Jira issue keys from the branch name and associated open merge request.
+ # Use BatchLoader to load this data without N+1 queries when serializing multiple branches
+ # in `Atlassian::JiraConnect::Serializers::BranchEntity`.
+ def issue_keys
+ BatchLoader.for(branch_name).batch do |branch_names, loader|
+ merge_requests = MergeRequest
+ .select(:description, :source_branch, :title)
+ .from_project(project)
+ .from_source_branches(branch_names)
+ .opened
+
+ branch_names.each do |branch_name|
+ related_merge_request = merge_requests.find { |mr| mr.source_branch == branch_name }
+
+ key_sources = [branch_name, related_merge_request&.title, related_merge_request&.description].compact
+ issue_keys = JiraIssueKeyExtractor.new(key_sources).issue_keys
+
+ loader.call(branch_name, issue_keys)
+ end
+ end
+ end
+
+ private
+
+ attr_reader :branch_name, :project
+ end
+ end
+end
diff --git a/lib/backup/database.rb b/lib/backup/database.rb
index bd3832c7327..28bc78a3932 100644
--- a/lib/backup/database.rb
+++ b/lib/backup/database.rb
@@ -27,7 +27,7 @@ module Backup
def dump(destination_dir, backup_id)
FileUtils.mkdir_p(destination_dir)
- snapshot_ids.each do |database_name, snapshot_id|
+ each_database_snapshot_id do |database_name, snapshot_id|
base_model = base_models_for_backup[database_name]
config = base_model.connection_db_config.configuration_hash
@@ -41,7 +41,7 @@ module Backup
pg_env(config)
pgsql_args = ["--clean"] # Pass '--clean' to include 'DROP TABLE' statements in the DB dump.
pgsql_args << '--if-exists'
- pgsql_args << "--snapshot=#{snapshot_ids[database_name]}"
+ pgsql_args << "--snapshot=#{snapshot_id}" if snapshot_id
if Gitlab.config.backup.pg_schema
pgsql_args << '-n'
@@ -55,7 +55,7 @@ module Backup
success = Backup::Dump::Postgres.new.dump(pg_database, db_file_name, pgsql_args)
- base_model.connection.rollback_transaction
+ base_model.connection.rollback_transaction if snapshot_id
raise DatabaseBackupError.new(config, db_file_name) unless success
@@ -63,8 +63,10 @@ module Backup
progress.flush
end
ensure
- base_models_for_backup.each do |_database_name, base_model|
- Gitlab::Database::TransactionTimeoutSettings.new(base_model.connection).restore_timeouts
+ ::Gitlab::Database::EachDatabase.each_database_connection(
+ only: base_models_for_backup.keys, include_shared: false
+ ) do |connection, _|
+ Gitlab::Database::TransactionTimeoutSettings.new(connection).restore_timeouts
end
end
@@ -237,31 +239,42 @@ module Backup
private
def drop_tables(database_name)
+ puts_time 'Cleaning the database ... '.color(:blue)
+
if Rake::Task.task_defined? "gitlab:db:drop_tables:#{database_name}"
- puts_time 'Cleaning the database ... '.color(:blue)
Rake::Task["gitlab:db:drop_tables:#{database_name}"].invoke
- puts_time 'done'.color(:green)
- elsif Gitlab::Database.database_base_models.one?
- # In single database, we do not have rake tasks per database
- puts_time 'Cleaning the database ... '.color(:blue)
+ else
+ # In single database (single or two connections)
Rake::Task["gitlab:db:drop_tables"].invoke
- puts_time 'done'.color(:green)
end
+
+ puts_time 'done'.color(:green)
end
def pg_restore_cmd(database)
['psql', database]
end
- def snapshot_ids
- @snapshot_ids ||= base_models_for_backup.each_with_object({}) do |(database_name, base_model), snapshot_ids|
- Gitlab::Database::TransactionTimeoutSettings.new(base_model.connection).disable_timeouts
+ def each_database_snapshot_id(&block)
+ @database_to_snapshot_id = {}
+
+ if @database_to_snapshot_id.empty?
+ ::Gitlab::Database::EachDatabase.each_database_connection(
+ only: base_models_for_backup.keys, include_shared: false
+ ) do |connection, database_name|
+ @database_to_snapshot_id[database_name] = nil
- base_model.connection.begin_transaction(isolation: :repeatable_read)
+ next unless Gitlab::Database.database_mode == Gitlab::Database::MODE_MULTIPLE_DATABASES
- snapshot_ids[database_name] =
- base_model.connection.execute("SELECT pg_export_snapshot() as snapshot_id;").first['snapshot_id']
+ Gitlab::Database::TransactionTimeoutSettings.new(connection).disable_timeouts
+
+ connection.begin_transaction(isolation: :repeatable_read)
+
+ @database_to_snapshot_id[database_name] = connection.select_value("SELECT pg_export_snapshot()")
+ end
end
+
+ @database_to_snapshot_id.each(&block)
end
end
end
diff --git a/lib/backup/gitaly_backup.rb b/lib/backup/gitaly_backup.rb
index 57dd74c7950..53c998efd71 100644
--- a/lib/backup/gitaly_backup.rb
+++ b/lib/backup/gitaly_backup.rb
@@ -9,14 +9,15 @@ module Backup
# @param [StringIO] progress IO interface to output progress
# @param [Integer] max_parallelism max parallelism when running backups
# @param [Integer] storage_parallelism max parallelism per storage (is affected by max_parallelism)
- def initialize(progress, max_parallelism: nil, storage_parallelism: nil, incremental: false, backup_id: nil)
+ # @param [Boolean] incremental if incremental backups should be created.
+ def initialize(progress, max_parallelism: nil, storage_parallelism: nil, incremental: false)
@progress = progress
@max_parallelism = max_parallelism
@storage_parallelism = storage_parallelism
@incremental = incremental
end
- def start(type, backup_repos_path, backup_id: nil)
+ def start(type, backup_repos_path, backup_id: nil, remove_all_repositories: nil)
raise Error, 'already started' if started?
if type == :create && !incremental?
@@ -35,9 +36,13 @@ module Backup
args = ['-layout', 'pointer']
args += ['-parallel', @max_parallelism.to_s] if @max_parallelism
args += ['-parallel-storage', @storage_parallelism.to_s] if @storage_parallelism
- if type == :create
+
+ case type
+ when :create
args += ['-incremental'] if incremental?
args += ['-id', backup_id] if backup_id
+ when :restore
+ args += ['-remove-all-repositories', remove_all_repositories.join(',')] if remove_all_repositories
end
@input_stream, stdout, @thread = Open3.popen2(build_env, bin_path, command, '-path', backup_repos_path, *args)
@@ -77,11 +82,7 @@ module Backup
#
# @see https://gitlab.com/gitlab-org/gitaly/-/blob/master/doc/gitaly-backup.md
def schedule_backup_job(repository, always_create:)
- connection_params = Gitlab::GitalyClient.connection_data(repository.storage)
-
json_job = {
- address: connection_params['address'],
- token: connection_params['token'],
storage_name: repository.storage,
relative_path: repository.relative_path,
gl_project_path: repository.gl_project_path,
@@ -91,10 +92,21 @@ module Backup
@input_stream.puts(json_job)
end
+ def gitaly_servers
+ Gitlab.config.repositories.storages.keys.index_with do |storage_name|
+ Gitlab::GitalyClient.connection_data(storage_name)
+ end
+ end
+
+ def gitaly_servers_encoded
+ Base64.strict_encode64(Gitlab::Json.dump(gitaly_servers))
+ end
+
def build_env
{
'SSL_CERT_FILE' => Gitlab::X509::Certificate.default_cert_file,
- 'SSL_CERT_DIR' => Gitlab::X509::Certificate.default_cert_dir
+ 'SSL_CERT_DIR' => Gitlab::X509::Certificate.default_cert_dir,
+ 'GITALY_SERVERS' => gitaly_servers_encoded
}.merge(ENV)
end
diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb
index ba4a26ba714..b5e1634004a 100644
--- a/lib/backup/manager.rb
+++ b/lib/backup/manager.rb
@@ -15,6 +15,10 @@ module Backup
repositories_paths: 'REPOSITORIES_PATHS'
}.freeze
+ YAML_PERMITTED_CLASSES = [
+ ActiveSupport::TimeWithZone, ActiveSupport::TimeZone, Symbol, Time
+ ].freeze
+
TaskDefinition = Struct.new(
:enabled, # `true` if the task can be used. Treated as `true` when not specified.
:human_name, # Name of the task used for logging.
@@ -247,7 +251,9 @@ module Backup
end
def read_backup_information
- @backup_information ||= YAML.load_file(File.join(backup_path, MANIFEST_NAME))
+ @backup_information ||= YAML.safe_load_file(
+ File.join(backup_path, MANIFEST_NAME),
+ permitted_classes: YAML_PERMITTED_CLASSES)
end
def write_backup_information
@@ -416,6 +422,12 @@ module Backup
end
end
+ def puts_available_timestamps
+ available_timestamps.each do |available_timestamp|
+ puts_time " " + available_timestamp
+ end
+ end
+
def unpack(source_backup_id)
if source_backup_id.blank? && non_tarred_backup?
puts_time "Non tarred backup found in #{backup_path}, using that"
@@ -431,7 +443,7 @@ module Backup
elsif backup_file_list.many? && source_backup_id.nil?
puts_time 'Found more than one backup:'
# print list of available backups
- puts_time " " + available_timestamps.join("\n ")
+ puts_available_timestamps
if incremental?
puts_time 'Please specify which one you want to create an incremental backup for:'
diff --git a/lib/backup/repositories.rb b/lib/backup/repositories.rb
index 4f4a098f374..218df3fcb6c 100644
--- a/lib/backup/repositories.rb
+++ b/lib/backup/repositories.rb
@@ -30,7 +30,7 @@ module Backup
override :restore
def restore(destination_path)
- strategy.start(:restore, destination_path)
+ strategy.start(:restore, destination_path, remove_all_repositories: remove_all_repositories)
enqueue_consecutive
ensure
@@ -44,6 +44,12 @@ module Backup
attr_reader :strategy, :storages, :paths
+ def remove_all_repositories
+ return if paths.present?
+
+ storages.presence || Gitlab.config.repositories.storages.keys
+ end
+
def enqueue_consecutive
enqueue_consecutive_projects
enqueue_consecutive_snippets
diff --git a/lib/banzai/filter/asset_proxy_filter.rb b/lib/banzai/filter/asset_proxy_filter.rb
index 6371a8f23af..00ffdd3d809 100644
--- a/lib/banzai/filter/asset_proxy_filter.rb
+++ b/lib/banzai/filter/asset_proxy_filter.rb
@@ -62,7 +62,7 @@ module Banzai
# whenever the application settings are changed
def self.initialize_settings
application_settings = Gitlab::CurrentSettings.current_application_settings
- Gitlab.config['asset_proxy'] ||= Settingslogic.new({})
+ Gitlab.config['asset_proxy'] ||= GitlabSettings::Options.build({})
if application_settings.respond_to?(:asset_proxy_enabled)
Gitlab.config.asset_proxy['enabled'] = application_settings.asset_proxy_enabled
diff --git a/lib/banzai/filter/base_sanitization_filter.rb b/lib/banzai/filter/base_sanitization_filter.rb
index 3b00d1a9824..0735fbb8d4c 100644
--- a/lib/banzai/filter/base_sanitization_filter.rb
+++ b/lib/banzai/filter/base_sanitization_filter.rb
@@ -25,7 +25,8 @@ module Banzai
# Allow data-math-style attribute in order to support LaTeX formatting
allowlist[:attributes]['code'] = %w(data-math-style)
- allowlist[:attributes]['pre'] = %w(data-math-style data-mermaid-style data-kroki-style)
+ allowlist[:attributes]['pre'] = %w(data-canonical-lang data-lang-params
+ data-math-style data-mermaid-style data-kroki-style)
# Allow html5 details/summary elements
allowlist[:elements].push('details')
diff --git a/lib/banzai/filter/blockquote_fence_filter.rb b/lib/banzai/filter/blockquote_fence_filter.rb
index 8681173b1f4..d4ff7d4c6b5 100644
--- a/lib/banzai/filter/blockquote_fence_filter.rb
+++ b/lib/banzai/filter/blockquote_fence_filter.rb
@@ -2,7 +2,7 @@
module Banzai
module Filter
- class BlockquoteFenceFilter < HTML::Pipeline::TextFilter
+ class BlockquoteFenceFilter < TimeoutTextPipelineFilter
REGEX = %r{
#{::Gitlab::Regex.markdown_code_or_html_blocks}
|
@@ -39,7 +39,7 @@ module Banzai
@text = @text.delete("\r")
end
- def call
+ def call_with_timeout
@text.gsub(REGEX) do
if $~[:blockquote]
# keep the same number of source lines/positions by replacing the
diff --git a/lib/banzai/filter/code_language_filter.rb b/lib/banzai/filter/code_language_filter.rb
new file mode 100644
index 00000000000..60e5a4063df
--- /dev/null
+++ b/lib/banzai/filter/code_language_filter.rb
@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module Banzai
+ module Filter
+ # HTML Filter to convert use of `lang` attribute into a common format,
+ # data-canonical-lang, as the `lang` attribute is really meant for accessibility
+ # and not for specifying code highlight language.
+ # See https://developer.mozilla.org/en-US/docs/Web/HTML/Global_attributes/lang#accessibility
+ # This also provides one place to transform the language specification format, whether it
+ # sits on the `pre` or `code`, or in a `class` or `lang` attribute
+ class CodeLanguageFilter < HTML::Pipeline::Filter
+ include OutputSafety
+
+ LANG_PARAMS_DELIMITER = ':'
+ LANG_ATTR = 'data-canonical-lang'
+ LANG_PARAMS_ATTR = 'data-lang-params'
+
+ CSS = 'pre > code:only-child'
+ XPATH = Gitlab::Utils::Nokogiri.css_to_xpath(CSS).freeze
+
+ def call
+ doc.xpath(XPATH).each do |node|
+ transform_node(node)
+ end
+
+ doc
+ end
+
+ def transform_node(code_node)
+ return if code_node.parent&.parent.nil?
+
+ lang, lang_params = parse_lang_params(code_node)
+ pre_node = code_node.parent
+
+ pre_node.remove_attribute('lang') if lang.present?
+ pre_node.set_attribute(LANG_ATTR, escape_once(lang)) if lang.present?
+ pre_node.set_attribute(LANG_PARAMS_ATTR, escape_once(lang_params)) if lang_params.present?
+
+ # cmark-gfm added this, it's now in data-lang-params
+ pre_node.remove_attribute('data-meta')
+ end
+
+ private
+
+ # cmark-gfm's FULL_INFO_STRING render option works with the space delimiter.
+ # Which means the language specified on a code block is parsed with spaces. Anything
+ # after the first space is placed in the `data-meta` attribute.
+ # However GitLab recognizes `:` as an additional delimiter on the lang attribute.
+ # So parse out the extra parameter.
+ #
+ # Original
+ # "```suggestion:+1-10 more```" -> '<pre data-canonical-lang="suggestion:+1-10" data-lang-params="more">'.
+ #
+ # With extra parsing
+ # "```suggestion:+1-10 more```" -> '<pre data-canonical-lang="suggestion" data-lang-params="+1-10 more">'.
+ def parse_lang_params(code_node)
+ pre_node = code_node.parent
+ language = pre_node.attr('lang')
+
+ return unless language
+
+ language, language_params = language.split(LANG_PARAMS_DELIMITER, 2)
+
+ # cmark-gfm places extra lang parameters into data-meta
+ language_params = [pre_node.attr('data-meta'), language_params].compact.join(' ')
+
+ [language, language_params]
+ end
+ end
+ end
+end
diff --git a/lib/banzai/filter/commit_trailers_filter.rb b/lib/banzai/filter/commit_trailers_filter.rb
index 817bea42757..c0160680a61 100644
--- a/lib/banzai/filter/commit_trailers_filter.rb
+++ b/lib/banzai/filter/commit_trailers_filter.rb
@@ -70,7 +70,7 @@ module Banzai
#
# Returns a String with a link to the user.
def link_to_user_or_email(name, email, trailer)
- link_to_user User.find_by_any_email(email),
+ link_to_user User.with_public_email(email).first,
name: name,
email: email,
trailer: trailer
diff --git a/lib/banzai/filter/dollar_math_pre_filter.rb b/lib/banzai/filter/dollar_math_pre_filter.rb
index aaa186f87a6..937328a2056 100644
--- a/lib/banzai/filter/dollar_math_pre_filter.rb
+++ b/lib/banzai/filter/dollar_math_pre_filter.rb
@@ -16,31 +16,30 @@ module Banzai
# by converting it into the ```math syntax. In this way, we can ensure
# that it's considered a code block and will not have any markdown processed inside it.
- # Corresponds to the "$$\n...\n$$" syntax
- REGEX = %r{
- #{::Gitlab::Regex.markdown_code_or_html_blocks}
- |
- (?=(?<=^\n|\A)\$\$\ *\n.*\n\$\$\ *(?=\n$|\z))(?:
- # Display math block:
- # $$
- # latex math
- # $$
-
- (?<=^\n|\A)\$\$\ *\n
- (?<display_math>
- (?:.)+?
- )
- \n\$\$\ *(?=\n$|\z)
- )
- }mx.freeze
+ # Display math block:
+ # $$
+ # latex math
+ # $$
+ REGEX =
+ "#{::Gitlab::Regex.markdown_code_or_html_blocks_or_html_comments_untrusted}" \
+ '|' \
+ '^\$\$\ *\n' \
+ '(?P<display_math>' \
+ '(?:\n|.)*?' \
+ ')' \
+ '\n\$\$\ *$' \
+ .freeze
def call
- @text.gsub(REGEX) do
- if $~[:display_math]
- # change from $$ to ```math
- "```math\n#{$~[:display_math]}\n```"
+ regex = Gitlab::UntrustedRegexp.new(REGEX, multiline: true)
+ return @text unless regex.match?(@text)
+
+ regex.replace_gsub(@text) do |match|
+ # change from $$ to ```math
+ if match[:display_math]
+ "```math\n#{match[:display_math]}\n```"
else
- $~[0]
+ match.to_s
end
end
end
diff --git a/lib/banzai/filter/inline_embeds_filter.rb b/lib/banzai/filter/inline_embeds_filter.rb
index c1077674cf0..a16166123f8 100644
--- a/lib/banzai/filter/inline_embeds_filter.rb
+++ b/lib/banzai/filter/inline_embeds_filter.rb
@@ -10,6 +10,8 @@ module Banzai
# the link, and insert this node after any html content
# surrounding the link.
def call
+ return doc if Feature.enabled?(:remove_monitor_metrics)
+
doc.xpath(xpath_search).each do |node|
next unless element = element_to_embed(node)
diff --git a/lib/banzai/filter/inline_observability_filter.rb b/lib/banzai/filter/inline_observability_filter.rb
index 50d4aac70cc..8e38f689959 100644
--- a/lib/banzai/filter/inline_observability_filter.rb
+++ b/lib/banzai/filter/inline_observability_filter.rb
@@ -1,14 +1,22 @@
# frozen_string_literal: true
-require 'uri'
-
module Banzai
module Filter
- class InlineObservabilityFilter < ::Banzai::Filter::InlineEmbedsFilter
+ class InlineObservabilityFilter < HTML::Pipeline::Filter
+ include Gitlab::Utils::StrongMemoize
+
def call
- return doc unless can_view_observability?
+ return doc unless Gitlab::Observability.enabled?(group)
+
+ doc.xpath(xpath_search).each do |node|
+ next unless element = element_to_embed(node)
+
+ # We want this to follow any surrounding content. For example,
+ # if a link is inline in a paragraph.
+ node.parent.children.last.add_next_sibling(element)
+ end
- super
+ doc
end
# Placeholder element for the frontend to use as an
@@ -17,40 +25,39 @@ module Banzai
doc.document.create_element(
'div',
class: 'js-render-observability',
- 'data-frame-url': url,
- 'data-observability-url': Gitlab::Observability.observability_url
+ 'data-frame-url': url
)
end
# Search params for selecting observability links.
def xpath_search
- "descendant-or-self::a[starts-with(@href, '#{Gitlab::Observability.observability_url}')]"
+ "descendant-or-self::a[starts-with(@href, '#{gitlab_domain}/groups/') and contains(@href,'/-/observability/')]"
end
# Creates a new element based on the parameters
# obtained from the target link
def element_to_embed(node)
url = node['href']
- uri = URI.parse(url)
- observability_uri = URI.parse(Gitlab::Observability.observability_url)
-
- if uri.scheme == observability_uri.scheme &&
- uri.port == observability_uri.port &&
- uri.host.casecmp?(observability_uri.host) &&
- uri.path.downcase.exclude?("auth/start")
- create_element(url)
- end
+
+ embeddable_url = extract_embeddable_url(url)
+ create_element(embeddable_url) if embeddable_url
end
private
- def can_view_observability?
- Feature.enabled?(:observability_group_tab, group)
+ def extract_embeddable_url(url)
+ strong_memoize_with(:embeddable_url, url) do
+ Gitlab::Observability.embeddable_url(url)
+ end
end
def group
context[:group] || context[:project]&.group
end
+
+ def gitlab_domain
+ ::Gitlab.config.gitlab.url
+ end
end
end
end
diff --git a/lib/banzai/filter/issuable_reference_expansion_filter.rb b/lib/banzai/filter/issuable_reference_expansion_filter.rb
index 6822e36c9be..ec7778a3630 100644
--- a/lib/banzai/filter/issuable_reference_expansion_filter.rb
+++ b/lib/banzai/filter/issuable_reference_expansion_filter.rb
@@ -10,13 +10,17 @@ module Banzai
class IssuableReferenceExpansionFilter < HTML::Pipeline::Filter
include Gitlab::Utils::StrongMemoize
+ NUMBER_OF_SUMMARY_ASSIGNEES = 2
VISIBLE_STATES = %w(closed merged).freeze
+ EXTENDED_FORMAT_XPATH = Gitlab::Utils::Nokogiri.css_to_xpath('a[data-reference-format="+s"]')
def call
return doc unless context[:issuable_reference_expansion_enabled]
- context = RenderContext.new(project, current_user)
- extractor = Banzai::IssuableExtractor.new(context)
+ options = { extended_preload: doc.xpath(EXTENDED_FORMAT_XPATH).present? }
+ extractor_context = RenderContext.new(project, current_user, options: options)
+
+ extractor = Banzai::IssuableExtractor.new(extractor_context)
issuables = extractor.extract([doc])
issuables.each do |node, issuable|
@@ -26,6 +30,9 @@ module Banzai
case node.attr('data-reference-format')
when '+'
expand_reference_with_title_and_state(node, issuable)
+ when '+s'
+ expand_reference_with_title_and_state(node, issuable)
+ expand_reference_with_summary(node, issuable)
else
expand_reference_with_state(node, issuable)
end
@@ -43,17 +50,47 @@ module Banzai
node.content += ')'
end
+ # rubocop:disable Style/AsciiComments
+ # Example: Issue Title (#123 - closed) assignee name 1, assignee name 2+ • v15.9 • On track
+ def expand_reference_with_summary(node, issuable)
+ summary = []
+
+ summary << assignees_text(issuable) if issuable.supports_assignee?
+ summary << milestone_text(issuable.milestone) if issuable.supports_milestone?
+ summary << health_status_text(issuable.health_status) if issuable.supports_health_status?
+
+ node.content = [node.content, *summary].compact_blank.join(' • ')
+ end
+ # rubocop:enable Style/AsciiComments
+
# Example: #123 (closed)
def expand_reference_with_state(node, issuable)
node.content += " (#{issuable_state_text(issuable)})"
end
+ def assignees_text(issuable)
+ assignee_names = issuable.assignees.first(NUMBER_OF_SUMMARY_ASSIGNEES + 1).map(&:sanitize_name)
+
+ return _('Unassigned') if assignee_names.empty?
+
+ "#{assignee_names.first(NUMBER_OF_SUMMARY_ASSIGNEES).to_sentence(two_words_connector: ', ')}" \
+ "#{assignee_names.size > NUMBER_OF_SUMMARY_ASSIGNEES ? '+' : ''}"
+ end
+
+ def milestone_text(milestone)
+ milestone&.title
+ end
+
+ def health_status_text(health_status)
+ health_status&.humanize
+ end
+
def issuable_state_text(issuable)
moved_issue?(issuable) ? s_("IssuableStatus|moved") : issuable.state
end
def moved_issue?(issuable)
- issuable.instance_of?(Issue) && issuable.moved?
+ issuable.is_a?(Issue) && issuable.moved?
end
def should_expand?(node, issuable)
diff --git a/lib/banzai/filter/kroki_filter.rb b/lib/banzai/filter/kroki_filter.rb
index 2b9e2a22c11..04f1a1b4f3c 100644
--- a/lib/banzai/filter/kroki_filter.rb
+++ b/lib/banzai/filter/kroki_filter.rb
@@ -18,8 +18,8 @@ module Banzai
diagram_selectors = ::Gitlab::Kroki.formats(settings)
.map do |diagram_type|
- %(pre[lang="#{diagram_type}"] > code,
- pre > code[lang="#{diagram_type}"])
+ %(pre[data-canonical-lang="#{diagram_type}"] > code,
+ pre > code[data-canonical-lang="#{diagram_type}"])
end
.join(', ')
@@ -28,7 +28,7 @@ module Banzai
diagram_format = "svg"
doc.xpath(xpath).each do |node|
- diagram_type = node.parent['lang'] || node['lang']
+ diagram_type = node.parent['data-canonical-lang'] || node['data-canonical-lang']
next unless diagram_selectors.include?(diagram_type)
diagram_src = node.content
diff --git a/lib/banzai/filter/markdown_engines/base.rb b/lib/banzai/filter/markdown_engines/base.rb
new file mode 100644
index 00000000000..34f1d4d3da9
--- /dev/null
+++ b/lib/banzai/filter/markdown_engines/base.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Banzai
+ module Filter
+ module MarkdownEngines
+ class Base
+ def initialize(context)
+ @context = context
+ end
+
+ def render(text)
+ raise NotImplementedError
+ end
+
+ private
+
+ def sourcepos_disabled?
+ @context[:no_sourcepos]
+ end
+ end
+ end
+ end
+end
diff --git a/lib/banzai/filter/markdown_engines/common_mark.rb b/lib/banzai/filter/markdown_engines/common_mark.rb
index 7abfadc612b..63680aa102c 100644
--- a/lib/banzai/filter/markdown_engines/common_mark.rb
+++ b/lib/banzai/filter/markdown_engines/common_mark.rb
@@ -9,7 +9,7 @@
module Banzai
module Filter
module MarkdownEngines
- class CommonMark
+ class CommonMark < Base
EXTENSIONS = [
:autolink, # provides support for automatically converting URLs to anchor tags.
:strikethrough, # provides support for strikethroughs.
@@ -29,9 +29,7 @@ module Banzai
:UNSAFE # allow raw/custom HTML and unsafe links.
].freeze
- def initialize(context)
- @context = context
- end
+ RENDER_OPTIONS_SOURCEPOS = RENDER_OPTIONS + [:SOURCEPOS].freeze
def render(text)
CommonMarker.render_html(text, render_options, EXTENSIONS)
@@ -40,17 +38,7 @@ module Banzai
private
def render_options
- @context[:no_sourcepos] ? render_options_no_sourcepos : render_options_sourcepos
- end
-
- def render_options_no_sourcepos
- RENDER_OPTIONS
- end
-
- def render_options_sourcepos
- render_options_no_sourcepos + [
- :SOURCEPOS # enable embedding of source position information
- ].freeze
+ sourcepos_disabled? ? RENDER_OPTIONS : RENDER_OPTIONS_SOURCEPOS
end
end
end
diff --git a/lib/banzai/filter/markdown_filter.rb b/lib/banzai/filter/markdown_filter.rb
index 242e39f5495..a546a72da5d 100644
--- a/lib/banzai/filter/markdown_filter.rb
+++ b/lib/banzai/filter/markdown_filter.rb
@@ -3,10 +3,12 @@
module Banzai
module Filter
class MarkdownFilter < HTML::Pipeline::TextFilter
+ DEFAULT_ENGINE = :common_mark
+
def initialize(text, context = nil, result = nil)
super(text, context, result)
- @renderer = renderer(context[:markdown_engine]).new(context)
+ @renderer = self.class.render_engine(context[:markdown_engine]).new(context)
@text = @text.delete("\r")
end
@@ -14,20 +16,20 @@ module Banzai
@renderer.render(@text).rstrip
end
- private
+ class << self
+ def render_engine(engine_from_context)
+ "Banzai::Filter::MarkdownEngines::#{engine(engine_from_context)}".constantize
+ rescue NameError
+ raise NameError, "`#{engine_from_context}` is unknown markdown engine"
+ end
- DEFAULT_ENGINE = :common_mark
+ private
- def engine(engine_from_context)
- engine_from_context ||= DEFAULT_ENGINE
-
- engine_from_context.to_s.classify
- end
+ def engine(engine_from_context)
+ engine_from_context ||= DEFAULT_ENGINE
- def renderer(engine_from_context)
- "Banzai::Filter::MarkdownEngines::#{engine(engine_from_context)}".constantize
- rescue NameError
- raise NameError, "`#{engine_from_context}` is unknown markdown engine"
+ engine_from_context.to_s.classify
+ end
end
end
end
diff --git a/lib/banzai/filter/math_filter.rb b/lib/banzai/filter/math_filter.rb
index 9b6fc71077a..e568f51652f 100644
--- a/lib/banzai/filter/math_filter.rb
+++ b/lib/banzai/filter/math_filter.rb
@@ -12,7 +12,7 @@ module Banzai
# Handle the $`...`$ and ```math syntax in this filter.
# Also add necessary classes any existing math blocks.
- CSS_MATH = 'pre[lang="math"] > code'
+ CSS_MATH = 'pre[data-canonical-lang="math"] > code'
XPATH_MATH = Gitlab::Utils::Nokogiri.css_to_xpath(CSS_MATH).freeze
CSS_CODE = 'code'
XPATH_CODE = Gitlab::Utils::Nokogiri.css_to_xpath(CSS_CODE).freeze
diff --git a/lib/banzai/filter/mermaid_filter.rb b/lib/banzai/filter/mermaid_filter.rb
index aaaf851ccf0..b562cbcdefe 100644
--- a/lib/banzai/filter/mermaid_filter.rb
+++ b/lib/banzai/filter/mermaid_filter.rb
@@ -4,7 +4,7 @@
module Banzai
module Filter
class MermaidFilter < HTML::Pipeline::Filter
- CSS = 'pre[lang="mermaid"] > code'
+ CSS = 'pre[data-canonical-lang="mermaid"] > code'
XPATH = Gitlab::Utils::Nokogiri.css_to_xpath(CSS).freeze
def call
diff --git a/lib/banzai/filter/plantuml_filter.rb b/lib/banzai/filter/plantuml_filter.rb
index 6a1fa64fb76..2e5f1b29c52 100644
--- a/lib/banzai/filter/plantuml_filter.rb
+++ b/lib/banzai/filter/plantuml_filter.rb
@@ -32,7 +32,7 @@ module Banzai
def lang_tag
@lang_tag ||= Gitlab::Utils::Nokogiri
- .css_to_xpath('pre[lang="plantuml"] > code, pre > code[lang="plantuml"]').freeze
+ .css_to_xpath('pre[data-canonical-lang="plantuml"] > code, pre > code[data-canonical-lang="plantuml"]').freeze
end
def settings
diff --git a/lib/banzai/filter/reference_redactor_filter.rb b/lib/banzai/filter/reference_redactor_filter.rb
index 485d3fd5fc7..9fae46a24a9 100644
--- a/lib/banzai/filter/reference_redactor_filter.rb
+++ b/lib/banzai/filter/reference_redactor_filter.rb
@@ -10,9 +10,9 @@ module Banzai
class ReferenceRedactorFilter < HTML::Pipeline::Filter
def call
unless context[:skip_redaction]
- context = RenderContext.new(project, current_user)
+ redactor_context = RenderContext.new(project, current_user)
- ReferenceRedactor.new(context).redact([doc])
+ ReferenceRedactor.new(redactor_context).redact([doc])
end
doc
diff --git a/lib/banzai/filter/references/abstract_reference_filter.rb b/lib/banzai/filter/references/abstract_reference_filter.rb
index 1ca38d2612d..3e48fe33b03 100644
--- a/lib/banzai/filter/references/abstract_reference_filter.rb
+++ b/lib/banzai/filter/references/abstract_reference_filter.rb
@@ -202,9 +202,13 @@ module Banzai
title = object_link_title(object, matches)
klass = reference_class(object_sym)
- data_attributes = data_attributes_for(link_content || match, parent, object,
- link_content: !!link_content,
- link_reference: link_reference)
+ data_attributes = data_attributes_for(
+ link_content || match,
+ parent,
+ object,
+ link_content: !!link_content,
+ link_reference: link_reference
+ )
data_attributes[:reference_format] = matches[:format] if matches.names.include?("format")
data_attributes.merge!(additional_object_attributes(object))
diff --git a/lib/banzai/filter/references/commit_range_reference_filter.rb b/lib/banzai/filter/references/commit_range_reference_filter.rb
index df7f42eaa70..d0a24f3f0f0 100644
--- a/lib/banzai/filter/references/commit_range_reference_filter.rb
+++ b/lib/banzai/filter/references/commit_range_reference_filter.rb
@@ -32,8 +32,7 @@ module Banzai
def url_for_object(range, project)
h = Gitlab::Routing.url_helpers
- h.project_compare_url(project,
- range.to_param.merge(only_path: context[:only_path]))
+ h.project_compare_url(project, range.to_param.merge(only_path: context[:only_path]))
end
def object_link_title(range, matches)
diff --git a/lib/banzai/filter/references/commit_reference_filter.rb b/lib/banzai/filter/references/commit_reference_filter.rb
index 86ab8597cf5..0f412c1fe8d 100644
--- a/lib/banzai/filter/references/commit_reference_filter.rb
+++ b/lib/banzai/filter/references/commit_reference_filter.rb
@@ -49,14 +49,14 @@ module Banzai
h = Gitlab::Routing.url_helpers
if referenced_merge_request_commit_shas.include?(commit.id)
- h.diffs_project_merge_request_url(project,
- noteable,
- commit_id: commit.id,
- only_path: only_path?)
+ h.diffs_project_merge_request_url(
+ project,
+ noteable,
+ commit_id: commit.id,
+ only_path: only_path?
+ )
else
- h.project_commit_url(project,
- commit,
- only_path: only_path?)
+ h.project_commit_url(project, commit, only_path: only_path?)
end
end
diff --git a/lib/banzai/filter/references/design_reference_filter.rb b/lib/banzai/filter/references/design_reference_filter.rb
index 01e1036dcec..16a2a2835e9 100644
--- a/lib/banzai/filter/references/design_reference_filter.rb
+++ b/lib/banzai/filter/references/design_reference_filter.rb
@@ -43,7 +43,7 @@ module Banzai
return [] unless project.design_management_enabled?
iids = identifiers.map(&:issue_iid).to_set
- issues = project.issues.where(iid: iids)
+ issues = project.issues.where(iid: iids).includes(:project, :namespace)
id_for_iid = issues.index_by(&:iid).transform_values(&:id)
issue_by_id = issues.index_by(&:id)
diff --git a/lib/banzai/filter/references/issue_reference_filter.rb b/lib/banzai/filter/references/issue_reference_filter.rb
index b536d900a02..9ad9d286ce3 100644
--- a/lib/banzai/filter/references/issue_reference_filter.rb
+++ b/lib/banzai/filter/references/issue_reference_filter.rb
@@ -22,7 +22,7 @@ module Banzai
end
def parent_records(parent, ids)
- parent.issues.where(iid: ids.to_a)
+ parent.issues.where(iid: ids.to_a).includes(:project, :namespace, :work_item_type)
end
def object_link_text_extras(issue, matches)
@@ -40,7 +40,7 @@ module Banzai
private
def additional_object_attributes(issue)
- { issue_type: issue.issue_type }
+ { issue_type: issue.work_item_type.base_type }
end
def issue_path(issue, project)
diff --git a/lib/banzai/filter/references/iteration_reference_filter.rb b/lib/banzai/filter/references/iteration_reference_filter.rb
deleted file mode 100644
index 591e07013c3..00000000000
--- a/lib/banzai/filter/references/iteration_reference_filter.rb
+++ /dev/null
@@ -1,15 +0,0 @@
-# frozen_string_literal: true
-
-module Banzai
- module Filter
- module References
- # The actual filter is implemented in the EE mixin
- class IterationReferenceFilter < AbstractReferenceFilter
- self.reference_type = :iteration
- self.object_class = Iteration
- end
- end
- end
-end
-
-Banzai::Filter::References::IterationReferenceFilter.prepend_mod_with('Banzai::Filter::References::IterationReferenceFilter')
diff --git a/lib/banzai/filter/references/merge_request_reference_filter.rb b/lib/banzai/filter/references/merge_request_reference_filter.rb
index 5bc18ee6985..2518d7653f6 100644
--- a/lib/banzai/filter/references/merge_request_reference_filter.rb
+++ b/lib/banzai/filter/references/merge_request_reference_filter.rb
@@ -13,8 +13,7 @@ module Banzai
def url_for_object(mr, project)
h = Gitlab::Routing.url_helpers
- h.project_merge_request_url(project, mr,
- only_path: context[:only_path])
+ h.project_merge_request_url(project, mr, only_path: context[:only_path])
end
def object_link_text_extras(object, matches)
diff --git a/lib/banzai/filter/references/snippet_reference_filter.rb b/lib/banzai/filter/references/snippet_reference_filter.rb
index 502bfca1ab7..1f5ab0645fe 100644
--- a/lib/banzai/filter/references/snippet_reference_filter.rb
+++ b/lib/banzai/filter/references/snippet_reference_filter.rb
@@ -23,8 +23,7 @@ module Banzai
def url_for_object(snippet, project)
h = Gitlab::Routing.url_helpers
- h.project_snippet_url(project, snippet,
- only_path: context[:only_path])
+ h.project_snippet_url(project, snippet, only_path: context[:only_path])
end
end
end
diff --git a/lib/banzai/filter/references/user_reference_filter.rb b/lib/banzai/filter/references/user_reference_filter.rb
index 1709b607c2e..5983036a8e5 100644
--- a/lib/banzai/filter/references/user_reference_filter.rb
+++ b/lib/banzai/filter/references/user_reference_filter.rb
@@ -139,11 +139,7 @@ module Banzai
end
def team_member?(user)
- if parent_group?
- parent.member?(user)
- else
- parent.team.member?(user)
- end
+ parent.member?(user)
end
def parent_url(link_content, author)
diff --git a/lib/banzai/filter/references/work_item_reference_filter.rb b/lib/banzai/filter/references/work_item_reference_filter.rb
new file mode 100644
index 00000000000..ed62b9a1be1
--- /dev/null
+++ b/lib/banzai/filter/references/work_item_reference_filter.rb
@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Banzai
+ module Filter
+ module References
+ # HTML filter that replaces work item references with links. References to
+ # work items that do not exist are ignored.
+ #
+ # This filter supports cross-project references.
+ class WorkItemReferenceFilter < IssueReferenceFilter
+ self.reference_type = :work_item
+ self.object_class = WorkItem
+
+ def parent_records(parent, ids)
+ parent.work_items.where(iid: ids.to_a)
+ end
+
+ private
+
+ def additional_object_attributes(work_item)
+ { work_item_type: work_item.work_item_type.base_type }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/banzai/filter/repository_link_filter.rb b/lib/banzai/filter/repository_link_filter.rb
index ddc3f5cf715..e06126bdf0f 100644
--- a/lib/banzai/filter/repository_link_filter.rb
+++ b/lib/banzai/filter/repository_link_filter.rb
@@ -204,7 +204,7 @@ module Banzai
end
def repo_visible_to_user?
- project && Ability.allowed?(current_user, :download_code, project)
+ project && Ability.allowed?(current_user, :read_code, project)
end
def ref
diff --git a/lib/banzai/filter/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb
index 3da6ce5c90c..e02d668a1ca 100644
--- a/lib/banzai/filter/syntax_highlight_filter.rb
+++ b/lib/banzai/filter/syntax_highlight_filter.rb
@@ -12,8 +12,6 @@ module Banzai
class SyntaxHighlightFilter < TimeoutHtmlPipelineFilter
include OutputSafety
- LANG_PARAMS_DELIMITER = ':'
- LANG_PARAMS_ATTR = 'data-lang-params'
CSS_CLASSES = 'code highlight js-syntax-highlight'
CSS = 'pre:not([data-kroki-style]) > code:only-child'
@@ -27,10 +25,13 @@ module Banzai
doc
end
- def highlight_node(node)
- return if node.parent&.parent.nil?
+ def highlight_node(code_node)
+ return if code_node.parent&.parent.nil?
- lang, lang_params = parse_lang_params(node)
+ # maintain existing attributes already added. e.g math and mermaid nodes
+ pre_node = code_node.parent
+
+ lang = pre_node['data-canonical-lang']
retried = false
if use_rouge?(lang)
@@ -42,7 +43,7 @@ module Banzai
end
begin
- code = Rouge::Formatters::HTMLGitlab.format(lex(lexer, node.text), tag: language)
+ code = Rouge::Formatters::HTMLGitlab.format(lex(lexer, code_node.text), tag: language)
rescue StandardError
# Gracefully handle syntax highlighter bugs/errors to ensure users can
# still access an issue/comment/etc. First, retry with the plain text
@@ -57,21 +58,16 @@ module Banzai
retry
end
- # maintain existing attributes already added. e.g math and mermaid nodes
- node.children = code
- pre_node = node.parent
+ code_node.children = code
# ensure there are no extra children, such as a text node that might
# show up from an XSS attack
- pre_node.children = node
+ pre_node.children = code_node
- pre_node[:lang] = language
pre_node.add_class(CSS_CLASSES)
pre_node.add_class("language-#{language}") if language
- pre_node.set_attribute('data-canonical-lang', escape_once(lang)) if lang != language
- pre_node.set_attribute(LANG_PARAMS_ATTR, escape_once(lang_params)) if lang_params.present?
+ pre_node.set_attribute('lang', language)
pre_node.set_attribute('v-pre', 'true')
- pre_node.remove_attribute('data-meta')
copy_code_btn = "<copy-code></copy-code>" unless language == 'suggestion'
highlighted = %(<div class="gl-relative markdown-code-block js-markdown-code">#{pre_node.to_html}#{copy_code_btn}</div>)
@@ -82,33 +78,6 @@ module Banzai
private
- def parse_lang_params(node)
- node = node.parent
-
- # Commonmarker's FULL_INFO_STRING render option works with the space delimiter.
- # But the current behavior of GitLab's markdown renderer is different - it grabs everything as the single
- # line, including language and its options. To keep backward compatibility, we have to parse the old format and
- # merge with the new one.
- #
- # Behaviors before separating language and its parameters:
- # Old ones:
- # "```ruby with options```" -> '<pre><code lang="ruby with options">'.
- # "```ruby:with:options```" -> '<pre><code lang="ruby:with:options">'.
- #
- # New ones:
- # "```ruby with options```" -> '<pre><code lang="ruby" data-meta="with options">'.
- # "```ruby:with:options```" -> '<pre><code lang="ruby:with:options">'.
-
- language = node.attr('lang')
-
- return unless language
-
- language, language_params = language.split(LANG_PARAMS_DELIMITER, 2)
- language_params = [node.attr('data-meta'), language_params].compact.join(' ')
-
- [language, language_params]
- end
-
# Separate method so it can be instrumented.
def lex(lexer, code)
lexer.lex(code)
diff --git a/lib/banzai/filter/timeout_html_pipeline_filter.rb b/lib/banzai/filter/timeout_html_pipeline_filter.rb
index b9b71163ab1..23bbeec8284 100644
--- a/lib/banzai/filter/timeout_html_pipeline_filter.rb
+++ b/lib/banzai/filter/timeout_html_pipeline_filter.rb
@@ -16,7 +16,6 @@ module Banzai
Gitlab::RenderTimeout.timeout(foreground: RENDER_TIMEOUT) { call_with_timeout }
rescue Timeout::Error => e
class_name = self.class.name.demodulize
- timeout_counter.increment(source: class_name)
Gitlab::ErrorTracking.track_exception(e, project_id: context[:project]&.id, class_name: class_name)
# we've timed out, but some work may have already been completed,
@@ -27,12 +26,6 @@ module Banzai
def call_with_timeout
raise NotImplementedError
end
-
- private
-
- def timeout_counter
- Gitlab::Metrics.counter(:banzai_filter_timeouts_total, 'Count of the Banzai filters that time out')
- end
end
end
end
diff --git a/lib/banzai/filter/timeout_text_pipeline_filter.rb b/lib/banzai/filter/timeout_text_pipeline_filter.rb
new file mode 100644
index 00000000000..318959065a2
--- /dev/null
+++ b/lib/banzai/filter/timeout_text_pipeline_filter.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Banzai
+ module Filter
+ # Text Filter that wraps a filter in a Gitlab::RenderTimeout.
+ # This way partial results can be returned, and the entire pipeline
+ # is not killed.
+ #
+ # This should not be used for any filter that must be allowed to complete,
+ # like a `ReferenceRedactorFilter`
+ #
+ class TimeoutTextPipelineFilter < HTML::Pipeline::TextFilter
+ RENDER_TIMEOUT = 10.seconds
+
+ def call
+ Gitlab::RenderTimeout.timeout(foreground: RENDER_TIMEOUT) { call_with_timeout }
+ rescue Timeout::Error => e
+ class_name = self.class.name.demodulize
+ Gitlab::ErrorTracking.track_exception(e, project_id: context[:project]&.id, class_name: class_name)
+
+ # we've timed out, but some work may have already been completed,
+ # so go ahead and return the text
+ @text
+ end
+
+ def call_with_timeout
+ raise NotImplementedError
+ end
+ end
+ end
+end
diff --git a/lib/banzai/issuable_extractor.rb b/lib/banzai/issuable_extractor.rb
index 34b6ca99e32..6428f71eb8f 100644
--- a/lib/banzai/issuable_extractor.rb
+++ b/lib/banzai/issuable_extractor.rb
@@ -12,6 +12,7 @@ module Banzai
attr_reader :context
ISSUE_REFERENCE_TYPE = '@data-reference-type="issue"'
+ WORK_ITEM_REFERENCE_TYPE = '@data-reference-type="work_item"'
MERGE_REQUEST_REFERENCE_TYPE = '@data-reference-type="merge_request"'
# context - An instance of Banzai::RenderContext.
@@ -41,6 +42,7 @@ module Banzai
def parsers
[
Banzai::ReferenceParser::IssueParser.new(context),
+ Banzai::ReferenceParser::WorkItemParser.new(context),
Banzai::ReferenceParser::MergeRequestParser.new(context)
]
end
@@ -53,7 +55,7 @@ module Banzai
end
def reference_types
- [ISSUE_REFERENCE_TYPE, MERGE_REQUEST_REFERENCE_TYPE]
+ [ISSUE_REFERENCE_TYPE, WORK_ITEM_REFERENCE_TYPE, MERGE_REQUEST_REFERENCE_TYPE]
end
end
end
diff --git a/lib/banzai/pipeline/ascii_doc_pipeline.rb b/lib/banzai/pipeline/ascii_doc_pipeline.rb
index 8764367426c..54306eadd41 100644
--- a/lib/banzai/pipeline/ascii_doc_pipeline.rb
+++ b/lib/banzai/pipeline/ascii_doc_pipeline.rb
@@ -6,6 +6,7 @@ module Banzai
def self.filters
FilterArray[
Filter::AsciiDocSanitizationFilter,
+ Filter::CodeLanguageFilter,
Filter::AssetProxyFilter,
Filter::ExternalLinkFilter,
Filter::PlantumlFilter,
diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb
index 6bd9e65f431..53f938c044f 100644
--- a/lib/banzai/pipeline/gfm_pipeline.rb
+++ b/lib/banzai/pipeline/gfm_pipeline.rb
@@ -11,6 +11,7 @@ module Banzai
# The GFM-to-HTML-to-GFM cycle is tested in spec/features/copy_as_gfm_spec.rb.
def self.filters
@filters ||= FilterArray[
+ Filter::CodeLanguageFilter,
Filter::PlantumlFilter,
# Must always be before the SanitizationFilter to prevent XSS attacks
Filter::SpacedLinkFilter,
@@ -57,6 +58,7 @@ module Banzai
Filter::References::ProjectReferenceFilter,
Filter::References::DesignReferenceFilter,
Filter::References::IssueReferenceFilter,
+ Filter::References::WorkItemReferenceFilter,
Filter::References::ExternalIssueReferenceFilter,
Filter::References::MergeRequestReferenceFilter,
Filter::References::SnippetReferenceFilter,
diff --git a/lib/banzai/pipeline/markup_pipeline.rb b/lib/banzai/pipeline/markup_pipeline.rb
index 635d4c0884e..cb421ff77b6 100644
--- a/lib/banzai/pipeline/markup_pipeline.rb
+++ b/lib/banzai/pipeline/markup_pipeline.rb
@@ -6,6 +6,7 @@ module Banzai
def self.filters
@filters ||= FilterArray[
Filter::SanitizationFilter,
+ Filter::CodeLanguageFilter,
Filter::AssetProxyFilter,
Filter::ExternalLinkFilter,
Filter::PlantumlFilter,
diff --git a/lib/banzai/reference_parser/commit_parser.rb b/lib/banzai/reference_parser/commit_parser.rb
index c51f4976c28..7c2bccc68d9 100644
--- a/lib/banzai/reference_parser/commit_parser.rb
+++ b/lib/banzai/reference_parser/commit_parser.rb
@@ -5,6 +5,8 @@ module Banzai
class CommitParser < BaseParser
self.reference_type = :commit
+ COMMITS_LIMIT = 1000
+
def referenced_by(nodes, options = {})
commit_ids = commit_ids_per_project(nodes)
projects = find_projects_for_hash_keys(commit_ids)
@@ -19,24 +21,11 @@ module Banzai
end
def find_commits(project, ids)
- commits = []
-
- return commits unless project.valid_repo?
-
- ids.each do |id|
- commit = project.commit(id)
-
- commits << commit if commit
- end
-
- commits
- end
+ return [] unless project.valid_repo?
- def nodes_visible_to_user(user, nodes)
- projects = lazy { projects_for_nodes(nodes) }
- user.preloaded_member_roles_for_projects(projects.values) if user
+ ids = ids.take(COMMITS_LIMIT)
- super
+ project.commits_by(oids: ids)
end
private
diff --git a/lib/banzai/reference_parser/commit_range_parser.rb b/lib/banzai/reference_parser/commit_range_parser.rb
index 3d09bc83151..fb4a392105f 100644
--- a/lib/banzai/reference_parser/commit_range_parser.rb
+++ b/lib/banzai/reference_parser/commit_range_parser.rb
@@ -38,13 +38,6 @@ module Banzai
range.valid_commits? ? range : nil
end
- def nodes_visible_to_user(user, nodes)
- projects = lazy { projects_for_nodes(nodes) }
- user.preloaded_member_roles_for_projects(projects.values) if user
-
- super
- end
-
private
def can_read_reference?(user, ref_project, node)
diff --git a/lib/banzai/reference_parser/issue_parser.rb b/lib/banzai/reference_parser/issue_parser.rb
index 6b1491cc56b..d0e74044bba 100644
--- a/lib/banzai/reference_parser/issue_parser.rb
+++ b/lib/banzai/reference_parser/issue_parser.rb
@@ -59,19 +59,30 @@ module Banzai
def records_for_nodes(nodes)
@issues_for_nodes ||= grouped_objects_for_nodes(
nodes,
- Issue.all.includes(
- :author,
- :assignees,
- {
- # These associations are primarily used for checking permissions.
- # Eager loading these ensures we don't end up running dozens of
- # queries in this process.
- project: [:namespace, :project_feature, :route]
- }
- ),
+ Issue.all.includes(node_includes),
self.class.data_attribute
)
end
+
+ private
+
+ def node_includes
+ includes = [
+ :work_item_type,
+ :namespace,
+ :author,
+ :assignees,
+ {
+ # These associations are primarily used for checking permissions.
+ # Eager loading these ensures we don't end up running dozens of
+ # queries in this process.
+ project: [:namespace, :project_feature, :route]
+ }
+ ]
+ includes << :milestone if context.options[:extended_preload]
+
+ includes
+ end
end
end
end
diff --git a/lib/banzai/reference_parser/iteration_parser.rb b/lib/banzai/reference_parser/iteration_parser.rb
deleted file mode 100644
index 981354aa8e1..00000000000
--- a/lib/banzai/reference_parser/iteration_parser.rb
+++ /dev/null
@@ -1,22 +0,0 @@
-# frozen_string_literal: true
-
-module Banzai
- module ReferenceParser
- # The actual parser is implemented in the EE mixin
- class IterationParser < BaseParser
- self.reference_type = :iteration
-
- def references_relation
- Iteration
- end
-
- private
-
- def can_read_reference?(_user, _ref_project, _node)
- false
- end
- end
- end
-end
-
-Banzai::ReferenceParser::IterationParser.prepend_mod_with('Banzai::ReferenceParser::IterationParser')
diff --git a/lib/banzai/reference_parser/merge_request_parser.rb b/lib/banzai/reference_parser/merge_request_parser.rb
index 3e28f06b783..2bd06e79e96 100644
--- a/lib/banzai/reference_parser/merge_request_parser.rb
+++ b/lib/banzai/reference_parser/merge_request_parser.rb
@@ -19,17 +19,21 @@ module Banzai
end
def records_for_nodes(nodes)
+ node_includes = [
+ :author,
+ :assignees,
+ {
+ # These associations are primarily used for checking permissions.
+ # Eager loading these ensures we don't end up running dozens of
+ # queries in this process.
+ target_project: [{ namespace: :route }, :project_feature, :route]
+ }
+ ]
+ node_includes << :milestone if context.options[:extended_preload]
+
@merge_requests_for_nodes ||= grouped_objects_for_nodes(
nodes,
- MergeRequest.includes(
- :author,
- :assignees,
- {
- # These associations are primarily used for checking permissions.
- # Eager loading these ensures we don't end up running dozens of
- # queries in this process.
- target_project: [{ namespace: :route }, :project_feature, :route]
- }),
+ MergeRequest.includes(node_includes),
self.class.data_attribute
)
end
diff --git a/lib/banzai/reference_parser/user_parser.rb b/lib/banzai/reference_parser/user_parser.rb
index c40ca9dc7cd..48e2bcc9a11 100644
--- a/lib/banzai/reference_parser/user_parser.rb
+++ b/lib/banzai/reference_parser/user_parser.rb
@@ -81,7 +81,7 @@ module Banzai
project = projects[node]
user = users[node]
- project && user ? project.team.member?(user) : false
+ project&.member?(user)
else
true
end
diff --git a/lib/banzai/reference_parser/work_item_parser.rb b/lib/banzai/reference_parser/work_item_parser.rb
new file mode 100644
index 00000000000..1ce0b067687
--- /dev/null
+++ b/lib/banzai/reference_parser/work_item_parser.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Banzai
+ module ReferenceParser
+ class WorkItemParser < IssueParser
+ self.reference_type = :work_item
+
+ def records_for_nodes(nodes)
+ @work_items_for_nodes ||= grouped_objects_for_nodes(
+ nodes,
+ WorkItem.all.includes(node_includes),
+ self.class.data_attribute
+ )
+ end
+ end
+ end
+end
diff --git a/lib/banzai/render_context.rb b/lib/banzai/render_context.rb
index e30fc9f469b..a69732a26e2 100644
--- a/lib/banzai/render_context.rb
+++ b/lib/banzai/render_context.rb
@@ -4,13 +4,14 @@ module Banzai
# Object storing the current user, project, and other details used when
# parsing Markdown references.
class RenderContext
- attr_reader :current_user
+ attr_reader :current_user, :options
# default_project - The default project to use for all documents, if any.
# current_user - The user viewing the document, if any.
- def initialize(default_project = nil, current_user = nil)
+ def initialize(default_project = nil, current_user = nil, options: {})
@current_user = current_user
@projects = Hash.new(default_project)
+ @options = options
end
# Associates an HTML document with a Project.
diff --git a/lib/banzai/renderer.rb b/lib/banzai/renderer.rb
index b16af78841a..b860fc0c6ae 100644
--- a/lib/banzai/renderer.rb
+++ b/lib/banzai/renderer.rb
@@ -21,10 +21,8 @@ module Banzai
cache_key = full_cache_key(cache_key, context[:pipeline])
if cache_key
- Gitlab::Metrics.measure(:banzai_cached_render) do
- Rails.cache.fetch(cache_key) do
- cacheless_render(text, context)
- end
+ Rails.cache.fetch(cache_key) do
+ cacheless_render(text, context)
end
else
cacheless_render(text, context)
@@ -160,40 +158,14 @@ module Banzai
def self.cacheless_render(text, context = {})
return text.to_s unless text.present?
- real_start = Gitlab::Metrics::System.monotonic_time
- cpu_start = Gitlab::Metrics::System.cpu_time
-
result = render_result(text, context)
output = result[:output]
- rendered = if output.respond_to?(:to_html)
- output.to_html
- else
- output.to_s
- end
-
- cpu_duration_histogram.observe({}, Gitlab::Metrics::System.cpu_time - cpu_start)
- real_duration_histogram.observe({}, Gitlab::Metrics::System.monotonic_time - real_start)
-
- rendered
- end
-
- def self.real_duration_histogram
- Gitlab::Metrics.histogram(
- :gitlab_banzai_cacheless_render_real_duration_seconds,
- 'Duration of Banzai pipeline rendering in real time',
- {},
- [0.01, 0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10.0, 50, 100]
- )
- end
-
- def self.cpu_duration_histogram
- Gitlab::Metrics.histogram(
- :gitlab_banzai_cacheless_render_cpu_duration_seconds,
- 'Duration of Banzai pipeline rendering in cpu time',
- {},
- Gitlab::Metrics::EXECUTION_MEASUREMENT_BUCKETS
- )
+ if output.respond_to?(:to_html)
+ output.to_html
+ else
+ output.to_s
+ end
end
def self.full_cache_key(cache_key, pipeline_name)
diff --git a/lib/bulk_imports/clients/graphql.rb b/lib/bulk_imports/clients/graphql.rb
index e1e78f52801..94bbdfaa681 100644
--- a/lib/bulk_imports/clients/graphql.rb
+++ b/lib/bulk_imports/clients/graphql.rb
@@ -38,12 +38,9 @@ module BulkImports
@url = Gitlab::Utils.append_path(url, '/api/graphql')
@token = token
@client = Graphlient::Client.new(@url, options(http: HTTP))
- @compatible_instance_version = false
end
def execute(...)
- validate_instance_version!
-
client.execute(...)
end
@@ -57,19 +54,6 @@ module BulkImports
}
}.merge(extra)
end
-
- def validate_instance_version!
- return if @compatible_instance_version
-
- response = client.execute('{ metadata { version } }')
- version = Gitlab::VersionInfo.parse(response.data.metadata.version)
-
- if version.major < BulkImport::MIN_MAJOR_VERSION
- raise ::BulkImports::Error.unsupported_gitlab_version
- else
- @compatible_instance_version = true
- end
- end
end
end
end
diff --git a/lib/bulk_imports/clients/http.rb b/lib/bulk_imports/clients/http.rb
index 6c36875111b..616ab8754b4 100644
--- a/lib/bulk_imports/clients/http.rb
+++ b/lib/bulk_imports/clients/http.rb
@@ -9,6 +9,7 @@ module BulkImports
DEFAULT_PAGE = 1
DEFAULT_PER_PAGE = 30
PAT_ENDPOINT_MIN_VERSION = '15.5.0'
+ SIDEKIQ_REQUEST_TIMEOUT = 60
def initialize(url:, token:, page: DEFAULT_PAGE, per_page: DEFAULT_PER_PAGE, api_version: API_VERSION)
@url = url
@@ -84,6 +85,8 @@ module BulkImports
end
def validate_instance_version!
+ raise ::BulkImports::Error.invalid_url unless instance_version.valid?
+
return true unless instance_version.major < BulkImport::MIN_MAJOR_VERSION
raise ::BulkImports::Error.unsupported_gitlab_version
@@ -140,7 +143,7 @@ module BulkImports
follow_redirects: true,
resend_on_redirect: false,
limit: 2
- }
+ }.merge(request_timeout.to_h)
end
def request_query
@@ -151,6 +154,10 @@ module BulkImports
}
end
+ def request_timeout
+ { timeout: SIDEKIQ_REQUEST_TIMEOUT } if Gitlab::Runtime.sidekiq?
+ end
+
def with_error_handling
response = yield
@@ -158,6 +165,8 @@ module BulkImports
raise ::BulkImports::NetworkError.new("Unsuccessful response #{response.code} from #{response.request.path.path}. Body: #{response.parsed_response}", response: response)
+ rescue Gitlab::HTTP::BlockedUrlError => e
+ raise e
rescue *Gitlab::HTTP::HTTP_ERRORS => e
raise ::BulkImports::NetworkError, e
end
diff --git a/lib/bulk_imports/error.rb b/lib/bulk_imports/error.rb
index 4699d5eab5f..c40b4bc7f34 100644
--- a/lib/bulk_imports/error.rb
+++ b/lib/bulk_imports/error.rb
@@ -3,21 +3,35 @@
module BulkImports
class Error < StandardError
def self.unsupported_gitlab_version
- self.new("Unsupported GitLab version. Source instance must run GitLab version #{BulkImport::MIN_MAJOR_VERSION} " \
- "or later.")
+ self.new("Unsupported GitLab version. Minimum supported version is #{BulkImport::MIN_MAJOR_VERSION}.")
end
def self.scope_validation_failure
- self.new("Import aborted as the provided personal access token does not have the required 'api' scope or " \
- "is no longer valid.")
+ self.new("Personal access token does not have the required " \
+ "'api' scope or is no longer valid.")
end
def self.invalid_url
- self.new("Import aborted as it was not possible to connect to the provided GitLab instance URL.")
+ self.new("Invalid source URL. Enter only the base URL of the source GitLab instance.")
+ end
+
+ def self.destination_namespace_validation_failure(destination_namespace)
+ self.new("Import failed. Destination '#{destination_namespace}' is invalid, or you don't have permission.")
+ end
+
+ def self.destination_slug_validation_failure
+ self.new("Import failed. Destination URL " \
+ "#{Gitlab::Regex.oci_repository_path_regex_message}")
end
def self.destination_full_path_validation_failure(full_path)
- self.new("Import aborted as '#{full_path}' already exists. Change the destination and try again.")
+ self.new("Import failed. '#{full_path}' already exists. Change the destination and try again.")
+ end
+
+ def self.setting_not_enabled
+ self.new("Group import disabled on source or destination instance. " \
+ "Ask an administrator to enable it on both instances and try again."
+ )
end
end
end
diff --git a/lib/bulk_imports/features.rb b/lib/bulk_imports/features.rb
deleted file mode 100644
index 9fdceb03655..00000000000
--- a/lib/bulk_imports/features.rb
+++ /dev/null
@@ -1,15 +0,0 @@
-# frozen_string_literal: true
-
-module BulkImports
- module Features
- def self.project_migration_enabled?(destination_namespace = nil)
- if destination_namespace.present?
- root_ancestor = Namespace.find_by_full_path(destination_namespace)&.root_ancestor
-
- ::Feature.enabled?(:bulk_import_projects, root_ancestor)
- else
- ::Feature.enabled?(:bulk_import_projects)
- end
- end
- end
-end
diff --git a/lib/bulk_imports/groups/stage.rb b/lib/bulk_imports/groups/stage.rb
index 7a777f1c8e1..bc9d490162c 100644
--- a/lib/bulk_imports/groups/stage.rb
+++ b/lib/bulk_imports/groups/stage.rb
@@ -19,6 +19,8 @@ module BulkImports
# `maximum_source_version` is equal to 15.1.0, the pipeline will be executed when importing from source instances
# running versions 15.1.1 (patch), 15.1.0, 15.0.1, 15.0.0, 14.10.0, etc. And it won't be executed when the source
# instance version is 15.2.0, 15.2.1, 16.0.0, etc.
+ #
+ # SubGroup Entities must be imported in later stage than Project Entities to avoid `full_path` naming conflicts.
def config
{
@@ -30,10 +32,6 @@ module BulkImports
pipeline: BulkImports::Groups::Pipelines::GroupAttributesPipeline,
stage: 1
},
- subgroups: {
- pipeline: BulkImports::Groups::Pipelines::SubgroupEntitiesPipeline,
- stage: 1
- },
namespace_settings: {
pipeline: BulkImports::Groups::Pipelines::NamespaceSettingsPipeline,
stage: 1,
@@ -55,6 +53,11 @@ module BulkImports
pipeline: BulkImports::Common::Pipelines::BadgesPipeline,
stage: 1
},
+ subgroups: {
+ pipeline: BulkImports::Groups::Pipelines::SubgroupEntitiesPipeline,
+ stage: 2 # SubGroup Entities must be imported in later stage
+ # to Project Entities to avoid `full_path` naming conflicts.
+ },
boards: {
pipeline: BulkImports::Common::Pipelines::BoardsPipeline,
stage: 2
@@ -71,7 +74,7 @@ module BulkImports
end
def project_entities_pipeline
- if migrate_projects? && project_pipeline_available? && feature_flag_enabled?
+ if migrate_projects? && project_pipeline_available?
{
project_entities: {
pipeline: BulkImports::Groups::Pipelines::ProjectEntitiesPipeline,
@@ -90,12 +93,6 @@ module BulkImports
def project_pipeline_available?
@bulk_import.source_version_info >= BulkImport.min_gl_version_for_project_migration
end
-
- def feature_flag_enabled?
- destination_namespace = @bulk_import_entity.destination_namespace
-
- BulkImports::Features.project_migration_enabled?(destination_namespace)
- end
end
end
end
diff --git a/lib/bulk_imports/groups/transformers/group_attributes_transformer.rb b/lib/bulk_imports/groups/transformers/group_attributes_transformer.rb
index 19993629ff5..85b52117dbc 100644
--- a/lib/bulk_imports/groups/transformers/group_attributes_transformer.rb
+++ b/lib/bulk_imports/groups/transformers/group_attributes_transformer.rb
@@ -5,6 +5,8 @@ module BulkImports
module Transformers
class GroupAttributesTransformer
include BulkImports::VisibilityLevel
+ include BulkImports::PathNormalization
+ include BulkImports::Uniquify
# rubocop: disable Style/IfUnlessModifier
def transform(context, data)
@@ -14,9 +16,11 @@ module BulkImports
namespace = Namespace.find_by_full_path(import_entity.destination_namespace)
end
+ path = normalize_path(import_entity.destination_slug)
+
params = {
- 'name' => group_name(namespace, data),
- 'path' => import_entity.destination_slug.parameterize,
+ 'name' => uniquify(namespace, data['name'], :name),
+ 'path' => uniquify(namespace, path, :path),
'description' => data['description'],
'lfs_enabled' => data['lfs_enabled'],
'emails_disabled' => data['emails_disabled'],
@@ -57,22 +61,6 @@ module BulkImports
params.with_indifferent_access
end
# rubocop: enable Style/IfUnlessModifier
-
- private
-
- def group_name(namespace, data)
- if namespace.present?
- namespace_children_names = namespace.children.pluck(:name) # rubocop: disable CodeReuse/ActiveRecord
-
- if namespace_children_names.include?(data['name'])
- data['name'] = Uniquify.new(1).string(-> (counter) { "#{data['name']}(#{counter})" }) do |base|
- namespace_children_names.include?(base)
- end
- end
- end
-
- data['name']
- end
end
end
end
diff --git a/lib/bulk_imports/ndjson_pipeline.rb b/lib/bulk_imports/ndjson_pipeline.rb
index 05d724a5e42..3c392910c1f 100644
--- a/lib/bulk_imports/ndjson_pipeline.rb
+++ b/lib/bulk_imports/ndjson_pipeline.rb
@@ -16,8 +16,6 @@ module BulkImports
return unless relation_hash
- relation_definition = import_export_config.top_relation_tree(relation)
-
relation_object = deep_transform_relation!(relation_hash, relation, relation_definition) do |key, hash|
relation_factory.create(
relation_index: relation_index,
@@ -35,8 +33,29 @@ module BulkImports
relation_object
end
- def load(_, object)
- object&.save!
+ def load(_context, object)
+ return unless object
+
+ if object.new_record?
+ saver = Gitlab::ImportExport::Base::RelationObjectSaver.new(
+ relation_object: object,
+ relation_key: relation,
+ relation_definition: relation_definition,
+ importable: portable
+ )
+
+ saver.execute
+
+ capture_invalid_subrelations(saver.invalid_subrelations)
+ else
+ if object.invalid?
+ Gitlab::Import::Errors.merge_nested_errors(object)
+
+ raise(ActiveRecord::RecordInvalid, object)
+ end
+
+ object.save!
+ end
end
def deep_transform_relation!(relation_hash, relation_key, relation_definition, &block)
@@ -104,6 +123,22 @@ module BulkImports
def portable_class_sym
portable.class.to_s.downcase.to_sym
end
+
+ def relation_definition
+ import_export_config.top_relation_tree(relation)
+ end
+
+ def capture_invalid_subrelations(invalid_subrelations)
+ invalid_subrelations.each do |record|
+ BulkImports::Failure.create(
+ bulk_import_entity_id: tracker.entity.id,
+ pipeline_class: tracker.pipeline_name,
+ exception_class: 'RecordInvalid',
+ exception_message: record.errors.full_messages.to_sentence.truncate(255),
+ correlation_id_value: Labkit::Correlation::CorrelationId.current_or_new_id
+ )
+ end
+ end
end
end
end
diff --git a/lib/bulk_imports/path_normalization.rb b/lib/bulk_imports/path_normalization.rb
new file mode 100644
index 00000000000..dfeef330ff8
--- /dev/null
+++ b/lib/bulk_imports/path_normalization.rb
@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module BulkImports
+ module PathNormalization
+ private
+
+ def normalize_path(path)
+ path = path.parameterize.downcase
+ return path if path =~ Gitlab::Regex.oci_repository_path_regex
+
+ # remove invalid characters from end and start of path
+ delete_invalid_edge_characters(delete_invalid_edge_characters(path))
+ # remove invalid multiplied characters
+ delete_invalid_multiple_characters(path)
+ end
+
+ def delete_invalid_edge_characters(path)
+ path.reverse!
+ path.each_char do |char|
+ break path unless char.match(Gitlab::Regex.oci_repository_path_regex).nil?
+
+ path.delete_prefix!(char)
+ end
+ end
+
+ def delete_invalid_multiple_characters(path)
+ path.gsub!('-_', '-') if path.include?('-_')
+ path.gsub!('_-', '-') if path.include?('_-')
+ path
+ end
+ end
+end
diff --git a/lib/bulk_imports/projects/graphql/get_project_query.rb b/lib/bulk_imports/projects/graphql/get_project_query.rb
index a2d7094d570..a2cba2fcfc0 100644
--- a/lib/bulk_imports/projects/graphql/get_project_query.rb
+++ b/lib/bulk_imports/projects/graphql/get_project_query.rb
@@ -11,6 +11,7 @@ module BulkImports
query($full_path: ID!) {
project(fullPath: $full_path) {
id
+ name
visibility
created_at: createdAt
}
diff --git a/lib/bulk_imports/projects/pipelines/commit_notes_pipeline.rb b/lib/bulk_imports/projects/pipelines/commit_notes_pipeline.rb
new file mode 100644
index 00000000000..092b03cb7b7
--- /dev/null
+++ b/lib/bulk_imports/projects/pipelines/commit_notes_pipeline.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module BulkImports
+ module Projects
+ module Pipelines
+ class CommitNotesPipeline
+ include NdjsonPipeline
+
+ relation_name 'commit_notes'
+
+ extractor ::BulkImports::Common::Extractors::NdjsonExtractor, relation: relation
+ end
+ end
+ end
+end
diff --git a/lib/bulk_imports/projects/pipelines/references_pipeline.rb b/lib/bulk_imports/projects/pipelines/references_pipeline.rb
index 8f44f3ffe6a..f00a62edf51 100644
--- a/lib/bulk_imports/projects/pipelines/references_pipeline.rb
+++ b/lib/bulk_imports/projects/pipelines/references_pipeline.rb
@@ -59,7 +59,7 @@ module BulkImports
end
def object_has_reference?(object)
- object_body(object).include?(source_full_path)
+ object_body(object)&.include?(source_full_path)
end
def object_body(object)
diff --git a/lib/bulk_imports/projects/stage.rb b/lib/bulk_imports/projects/stage.rb
index 73e102696fa..eecd567f54f 100644
--- a/lib/bulk_imports/projects/stage.rb
+++ b/lib/bulk_imports/projects/stage.rb
@@ -104,6 +104,11 @@ module BulkImports
pipeline: BulkImports::Projects::Pipelines::CiPipelinesPipeline,
stage: 5
},
+ commit_notes: {
+ pipeline: BulkImports::Projects::Pipelines::CommitNotesPipeline,
+ minimum_source_version: '15.10.0',
+ stage: 5
+ },
wiki: {
pipeline: BulkImports::Common::Pipelines::WikiPipeline,
stage: 5
diff --git a/lib/bulk_imports/projects/transformers/project_attributes_transformer.rb b/lib/bulk_imports/projects/transformers/project_attributes_transformer.rb
index c5ed9d42e44..1e025e91038 100644
--- a/lib/bulk_imports/projects/transformers/project_attributes_transformer.rb
+++ b/lib/bulk_imports/projects/transformers/project_attributes_transformer.rb
@@ -5,6 +5,8 @@ module BulkImports
module Transformers
class ProjectAttributesTransformer
include BulkImports::VisibilityLevel
+ include BulkImports::PathNormalization
+ include BulkImports::Uniquify
PROJECT_IMPORT_TYPE = 'gitlab_project_migration'
@@ -12,13 +14,14 @@ module BulkImports
project = {}
entity = context.entity
namespace = Namespace.find_by_full_path(entity.destination_namespace)
+ path = normalize_path(entity.destination_slug)
- project[:name] = entity.destination_slug
- project[:path] = entity.destination_slug.parameterize
+ project[:name] = uniquify(namespace, data['name'], :name)
+ project[:path] = uniquify(namespace, path, :path)
project[:created_at] = data['created_at']
project[:import_type] = PROJECT_IMPORT_TYPE
project[:visibility_level] = visibility_level(entity, namespace, data['visibility'])
- project[:namespace_id] = namespace.id if namespace
+ project[:namespace_id] = namespace.id
project.with_indifferent_access
end
diff --git a/lib/bulk_imports/uniquify.rb b/lib/bulk_imports/uniquify.rb
new file mode 100644
index 00000000000..a4290eb86bf
--- /dev/null
+++ b/lib/bulk_imports/uniquify.rb
@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module BulkImports
+ module Uniquify
+ private
+
+ def uniquify(namespace, data_item, data_type)
+ return data_item unless namespace.present?
+
+ children_items = Set.new
+
+ # index_namespaces_on_parent_id_and_id index supports this
+ Namespace.by_parent(namespace).each_batch do |relation|
+ children_items.merge(relation.pluck(data_type).to_set) # rubocop: disable CodeReuse/ActiveRecord
+ end
+
+ return data_item unless children_items.include?(data_item)
+
+ data_item = Gitlab::Utils::Uniquify.new(1).string(->(counter) { "#{data_item}_#{counter}" }) do |base|
+ children_items.include?(base)
+ end
+ end
+ end
+end
diff --git a/lib/container_registry/gitlab_api_client.rb b/lib/container_registry/gitlab_api_client.rb
index 5dddd421223..00877bb5a48 100644
--- a/lib/container_registry/gitlab_api_client.rb
+++ b/lib/container_registry/gitlab_api_client.rb
@@ -22,6 +22,8 @@ module ContainerRegistry
REGISTRY_GITLAB_V1_API_FEATURE = 'gitlab_v1_api'
MAX_TAGS_PAGE_SIZE = 1000
+ MAX_REPOSITORIES_PAGE_SIZE = 1000
+ PAGE_SIZE = 1
UnsuccessfulResponseError = Class.new(StandardError)
@@ -37,6 +39,48 @@ module ContainerRegistry
end
end
+ def self.one_project_with_container_registry_tag(path)
+ with_dummy_client(token_config: { type: :nested_repositories_token, path: path&.downcase }) do |client|
+ page = client.sub_repositories_with_tag(path&.downcase, page_size: PAGE_SIZE)
+ details = page[:response_body]&.first
+
+ break unless details
+
+ path = ContainerRegistry::Path.new(details["path"])
+
+ break unless path.valid?
+
+ ContainerRepository.find_by_path(path)&.project
+ end
+ end
+
+ def self.each_sub_repositories_with_tag_page(path:, page_size: 100, &block)
+ raise ArgumentError, 'block not given' unless block
+
+ # dummy uri to initialize the loop
+ next_page_uri = URI('')
+ page_count = 0
+
+ with_dummy_client(token_config: { type: :nested_repositories_token, path: path&.downcase }) do |client|
+ while next_page_uri
+ last = Rack::Utils.parse_nested_query(next_page_uri.query)['last']
+ current_page = client.sub_repositories_with_tag(path&.downcase, page_size: page_size, last: last)
+
+ if current_page&.key?(:response_body)
+ yield (current_page[:response_body] || [])
+ next_page_uri = current_page.dig(:pagination, :next, :uri)
+ else
+ # no current page. Break the loop
+ next_page_uri = nil
+ end
+
+ page_count += 1
+
+ raise 'too many pages requested' if page_count >= MAX_REPOSITORIES_PAGE_SIZE
+ end
+ end
+ end
+
# https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs-gitlab/api.md#compliance-check
def supports_gitlab_api?
strong_memoize(:supports_gitlab_api) do
@@ -133,6 +177,37 @@ module ContainerRegistry
end
end
+ # https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs-gitlab/api.md#list-sub-repositories
+ def sub_repositories_with_tag(path, page_size: 100, last: nil)
+ limited_page_size = [page_size, MAX_REPOSITORIES_PAGE_SIZE].min
+
+ with_token_faraday do |faraday_client|
+ url = "/gitlab/v1/repository-paths/#{path}/repositories/list/"
+ response = faraday_client.get(url) do |req|
+ req.params['n'] = limited_page_size
+ req.params['last'] = last if last
+ end
+
+ unless response.success?
+ Gitlab::ErrorTracking.log_exception(
+ UnsuccessfulResponseError.new,
+ class: self.class.name,
+ url: url,
+ status_code: response.status
+ )
+
+ break {}
+ end
+
+ link_parser = Gitlab::Utils::LinkHeaderParser.new(response.headers['link'])
+
+ {
+ pagination: link_parser.parse,
+ response_body: response_body(response)
+ }
+ end
+ end
+
private
def start_import_for(path, pre:)
diff --git a/lib/error_tracking/sentry_client/token.rb b/lib/error_tracking/sentry_client/token.rb
new file mode 100644
index 00000000000..ce7db5f2800
--- /dev/null
+++ b/lib/error_tracking/sentry_client/token.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module ErrorTracking
+ class SentryClient
+ class Token
+ MASKED_TOKEN_REGEX = /\A\*+\z/
+
+ def self.masked_token?(token)
+ MASKED_TOKEN_REGEX.match?(token)
+ end
+ end
+ end
+end
diff --git a/lib/feature.rb b/lib/feature.rb
index 17c26796ea1..3847f880be0 100644
--- a/lib/feature.rb
+++ b/lib/feature.rb
@@ -40,19 +40,6 @@ module Feature
class << self
delegate :group, to: :flipper
- def feature_flags_available?
- # When the DBMS is not available, an exception (e.g. PG::ConnectionBad) is raised
- active_db_connection = begin
- ActiveRecord::Base.connection.active? # rubocop:disable Database/MultipleDatabases
- rescue StandardError
- false
- end
-
- active_db_connection && Feature::FlipperFeature.table_exists?
- rescue ActiveRecord::NoDatabaseError
- false
- end
-
def all
flipper.features.to_a
end
@@ -205,9 +192,9 @@ module Feature
# This method is called from config/initializers/flipper.rb and can be used
# to register Flipper groups.
# See https://docs.gitlab.com/ee/development/feature_flags/index.html
- def register_feature_groups
- Flipper.register(:gitlab_team_members) { |actor| FeatureGroups::GitlabTeamMembers.enabled?(actor.thing) }
- end
+ #
+ # EE feature groups should go inside the ee/lib/ee/feature.rb version of this method.
+ def register_feature_groups; end
def register_definitions
Feature::Definition.reload!
@@ -339,7 +326,7 @@ module Feature
end
def l2_cache_backend
- Rails.cache
+ ::Gitlab::Redis::FeatureFlag.cache_store
end
def log(key:, action:, **extra)
diff --git a/lib/feature/gitaly.rb b/lib/feature/gitaly.rb
index fd798862fa8..eadf98e090e 100644
--- a/lib/feature/gitaly.rb
+++ b/lib/feature/gitaly.rb
@@ -53,11 +53,19 @@ module Feature
end
def project_actor(container)
- ::Feature::Gitaly::ActorWrapper.new(::Project, container.id) if container.is_a?(::Project)
+ return actor_wrapper(::Project, container.id) if container.is_a?(::Project)
+ return actor_wrapper(::Project, container.project.id) if container.is_a?(DesignManagement::Repository)
end
def group_actor(container)
- ::Feature::Gitaly::ActorWrapper.new(::Group, container.namespace_id) if container.is_a?(::Project)
+ return actor_wrapper(::Group, container.namespace_id) if container.is_a?(::Project)
+ return actor_wrapper(::Group, container.project.namespace_id) if container.is_a?(DesignManagement::Repository)
+ end
+
+ private
+
+ def actor_wrapper(actor_type, id)
+ ::Feature::Gitaly::ActorWrapper.new(actor_type, id)
end
end
end
diff --git a/lib/feature/logger.rb b/lib/feature/logger.rb
index 95e160273b6..337d3a4ccc2 100644
--- a/lib/feature/logger.rb
+++ b/lib/feature/logger.rb
@@ -2,6 +2,8 @@
module Feature
class Logger < ::Gitlab::JsonLogger
+ exclude_context!
+
def self.file_name_noext
'features_json'
end
diff --git a/lib/feature_groups/gitlab_team_members.rb b/lib/feature_groups/gitlab_team_members.rb
deleted file mode 100644
index 7f4c597fddd..00000000000
--- a/lib/feature_groups/gitlab_team_members.rb
+++ /dev/null
@@ -1,31 +0,0 @@
-# frozen_string_literal: true
-
-module FeatureGroups
- class GitlabTeamMembers
- GITLAB_COM_GROUP_ID = 6543
-
- class << self
- def enabled?(thing)
- return false unless Gitlab.com?
-
- team_member?(thing)
- end
-
- private
-
- def team_member?(thing)
- thing.is_a?(::User) && gitlab_com_member_ids.include?(thing.id)
- end
-
- def gitlab_com
- @gitlab_com ||= ::Group.find(GITLAB_COM_GROUP_ID)
- end
-
- def gitlab_com_member_ids
- Rails.cache.fetch("gitlab_team_members", expires_in: 1.hour) do
- gitlab_com.members.pluck_user_ids.to_set
- end
- end
- end
- end
-end
diff --git a/lib/generators/batched_background_migration/USAGE b/lib/generators/batched_background_migration/USAGE
new file mode 100644
index 00000000000..2fc2b2f7b96
--- /dev/null
+++ b/lib/generators/batched_background_migration/USAGE
@@ -0,0 +1,12 @@
+Description:
+ Generates files required for batched background migration.
+
+Example:
+ rails g batched_background_migration my_batched_migration --table_name=users --column_name=id --feature_category=gitaly
+
+ This will create:
+ db/post_migrate/20230213215230_queue_my_batched_migration.rb
+ spec/migrations/20230213215230_queue_my_batched_migration_spec.rb
+ lib/gitlab/background_migration/my_batched_migration.rb
+ spec/lib/gitlab/background_migration/my_batched_migration_spec.rb
+ db/docs/batched_background_migrations/my_batched_migration.yml
diff --git a/lib/generators/batched_background_migration/batched_background_migration_generator.rb b/lib/generators/batched_background_migration/batched_background_migration_generator.rb
new file mode 100644
index 00000000000..c68ed52c1a0
--- /dev/null
+++ b/lib/generators/batched_background_migration/batched_background_migration_generator.rb
@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+require 'rails/generators/active_record'
+
+module BatchedBackgroundMigration
+ class BatchedBackgroundMigrationGenerator < ActiveRecord::Generators::Base
+ source_root File.expand_path('templates', __dir__)
+
+ class_option :table_name
+ class_option :column_name
+ class_option :feature_category
+
+ def validate!
+ raise ArgumentError, "table_name is required" unless table_name.present?
+ raise ArgumentError, "column_name is required" unless column_name.present?
+ raise ArgumentError, "feature_category is required" unless feature_category.present?
+ end
+
+ def create_post_migration_and_specs
+ migration_template(
+ "queue_batched_background_migration.template",
+ File.join(db_migrate_path, "queue_#{file_name}.rb")
+ )
+
+ template(
+ "queue_batched_background_migration_spec.template",
+ File.join("spec/migrations/#{migration_number}_queue_#{file_name}_spec.rb")
+ )
+ end
+
+ def create_batched_background_migration_class_and_specs
+ template(
+ "batched_background_migration_job.template",
+ File.join("lib/gitlab/background_migration/#{file_name}.rb")
+ )
+
+ template(
+ "batched_background_migration_job_spec.template",
+ File.join("spec/lib/gitlab/background_migration/#{file_name}_spec.rb")
+ )
+ end
+
+ def create_dictionary_file
+ template(
+ "batched_background_migration_dictionary.template",
+ File.join("db/docs/batched_background_migrations/#{file_name}.yml")
+ )
+ end
+
+ def db_migrate_path
+ super.sub("migrate", "post_migrate")
+ end
+
+ private
+
+ def table_name
+ options[:table_name]
+ end
+
+ def column_name
+ options[:column_name]
+ end
+
+ def feature_category
+ options[:feature_category]
+ end
+
+ def current_milestone
+ version = Gem::Version.new(File.read('VERSION'))
+ version.release.segments.first(2).join('.')
+ end
+ end
+end
diff --git a/lib/generators/batched_background_migration/templates/batched_background_migration_dictionary.template b/lib/generators/batched_background_migration/templates/batched_background_migration_dictionary.template
new file mode 100644
index 00000000000..8aa08e15f48
--- /dev/null
+++ b/lib/generators/batched_background_migration/templates/batched_background_migration_dictionary.template
@@ -0,0 +1,6 @@
+---
+migration_job_name: <%= class_name %>
+description: # Please capture what <%= class_name %> does
+feature_category: <%= feature_category %>
+introduced_by_url: # URL of the MR (or issue/commit) that introduced the migration
+milestone: <%= current_milestone %>
diff --git a/lib/generators/batched_background_migration/templates/batched_background_migration_job.template b/lib/generators/batched_background_migration/templates/batched_background_migration_job.template
new file mode 100644
index 00000000000..c57ac637cb8
--- /dev/null
+++ b/lib/generators/batched_background_migration/templates/batched_background_migration_job.template
@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+# See https://docs.gitlab.com/ee/development/database/batched_background_migrations.html
+# for more information on how to use batched background migrations
+
+# Update below commented lines with appropriate values.
+
+module Gitlab
+ module BackgroundMigration
+ class <%= class_name %> < BatchedMigrationJob
+ # operation_name :my_operation
+ # scope_to ->(relation) { relation.where(column: "value") }
+ feature_category :<%= feature_category %>
+
+ def perform
+ each_sub_batch do |sub_batch|
+ # Your action on each sub_batch
+ end
+ end
+ end
+ end
+end
diff --git a/lib/generators/batched_background_migration/templates/batched_background_migration_job_spec.template b/lib/generators/batched_background_migration/templates/batched_background_migration_job_spec.template
new file mode 100644
index 00000000000..c41b8107c95
--- /dev/null
+++ b/lib/generators/batched_background_migration/templates/batched_background_migration_job_spec.template
@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::BackgroundMigration::<%= class_name %>, schema: <%= migration_number %>, feature_category: :<%= feature_category %> do # rubocop:disable Layout/LineLength
+ # Tests go here
+end
diff --git a/lib/generators/batched_background_migration/templates/queue_batched_background_migration.template b/lib/generators/batched_background_migration/templates/queue_batched_background_migration.template
new file mode 100644
index 00000000000..502edf2c1d7
--- /dev/null
+++ b/lib/generators/batched_background_migration/templates/queue_batched_background_migration.template
@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+# See https://docs.gitlab.com/ee/development/database/batched_background_migrations.html
+# for more information on when/how to queue batched background migrations
+
+# Update below commented lines with appropriate values.
+
+class <%= migration_class_name %> < Gitlab::Database::Migration[<%= Gitlab::Database::Migration.current_version %>]
+ MIGRATION = "<%= class_name %>"
+ # DELAY_INTERVAL = 2.minutes
+ # BATCH_SIZE = <%= Gitlab::Database::Migrations::BatchedBackgroundMigrationHelpers::BATCH_SIZE %>
+ # SUB_BATCH_SIZE = <%= Gitlab::Database::Migrations::BatchedBackgroundMigrationHelpers::SUB_BATCH_SIZE %>
+
+ def up
+ queue_batched_background_migration(
+ MIGRATION,
+ :<%= table_name %>,
+ :<%= column_name %>,
+ job_interval: DELAY_INTERVAL,
+ batch_size: BATCH_SIZE,
+ sub_batch_size: SUB_BATCH_SIZE
+ )
+ end
+
+ def down
+ delete_batched_background_migration(MIGRATION, :<%= table_name.to_sym %>, :<%= column_name.to_sym %>, [])
+ end
+end
diff --git a/lib/generators/batched_background_migration/templates/queue_batched_background_migration_spec.template b/lib/generators/batched_background_migration/templates/queue_batched_background_migration_spec.template
new file mode 100644
index 00000000000..e0a3078114e
--- /dev/null
+++ b/lib/generators/batched_background_migration/templates/queue_batched_background_migration_spec.template
@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+require_migration!
+
+RSpec.describe <%= migration_class_name %>, feature_category: :<%= feature_category.to_sym %> do
+ # let!(:batched_migration) { described_class::MIGRATION }
+
+ # it 'schedules a new batched migration' do
+ # reversible_migration do |migration|
+ # migration.before -> {
+ # expect(batched_migration).not_to have_scheduled_batched_migration
+ # }
+
+ # migration.after -> {
+ # expect(batched_migration).to have_scheduled_batched_migration(
+ # table_name: :<%= table_name %>,
+ # column_name: :<%= column_name %>,
+ # interval: described_class::DELAY_INTERVAL,
+ # batch_size: described_class::BATCH_SIZE,
+ # sub_batch_size: described_class::SUB_BATCH_SIZE
+ # )
+ # }
+ # end
+ # end
+end
diff --git a/lib/generators/gitlab/snowplow_event_definition_generator.rb b/lib/generators/gitlab/snowplow_event_definition_generator.rb
index 827e87dc313..b1a31541350 100644
--- a/lib/generators/gitlab/snowplow_event_definition_generator.rb
+++ b/lib/generators/gitlab/snowplow_event_definition_generator.rb
@@ -14,12 +14,11 @@ module Gitlab
class_option :ee, type: :boolean, optional: true, default: false, desc: 'Indicates if event is for ee'
class_option :category, type: :string, optional: false, desc: 'Category of the event'
class_option :action, type: :string, optional: false, desc: 'Action of the event'
- class_option :force, type: :boolean, optional: true, default: false, desc: 'Overwrite existing definition'
def create_event_file
- raise "Event definition already exists at #{file_path}" if definition_exists? && !force_definition_override?
+ raise "Event definition already exists at #{file_path}" if definition_exists?
- template "event_definition.yml", file_path, force: force_definition_override?
+ template "event_definition.yml", file_path, force: false
end
def distributions
@@ -42,10 +41,6 @@ module Gitlab
options[:ee]
end
- def force_definition_override?
- options[:force]
- end
-
private
def definition_exists?
@@ -64,8 +59,10 @@ module Gitlab
File.join(EE_DIR, file_name)
end
+ # Example of file name
+ # 20230227000018_project_management_issue_title_changed.yml
def file_name
- name = remove_special_chars("#{Time.current.to_i}_#{event_category}_#{event_action}")
+ name = remove_special_chars("#{Time.now.utc.strftime('%Y%m%d%H%M%S')}_#{event_category}_#{event_action}")
"#{name[0..95]}.yml" # max 100 chars, see https://gitlab.com/gitlab-com/gl-infra/delivery/-/issues/2030#note_679501200
end
diff --git a/lib/gitlab.rb b/lib/gitlab.rb
index 1190c92ce17..0361adc7fc7 100644
--- a/lib/gitlab.rb
+++ b/lib/gitlab.rb
@@ -60,6 +60,10 @@ module Gitlab
simulate_com? || Gitlab.config.gitlab.url == Gitlab::Saas.com_url || gl_subdomain?
end
+ def self.com_except_jh?
+ com? && !jh?
+ end
+
def self.com
yield if com?
end
diff --git a/lib/gitlab/access.rb b/lib/gitlab/access.rb
index fa025a2658f..bafda11170a 100644
--- a/lib/gitlab/access.rb
+++ b/lib/gitlab/access.rb
@@ -16,6 +16,7 @@ module Gitlab
DEVELOPER = 30
MAINTAINER = 40
OWNER = 50
+ ADMIN = 60
# Branch protection settings
PROTECTION_NONE = 0
diff --git a/lib/gitlab/action_cable/request_store_callbacks.rb b/lib/gitlab/action_cable/request_store_callbacks.rb
index a9f30b0fc10..14d80a7c40c 100644
--- a/lib/gitlab/action_cable/request_store_callbacks.rb
+++ b/lib/gitlab/action_cable/request_store_callbacks.rb
@@ -5,8 +5,6 @@ module Gitlab
module RequestStoreCallbacks
def self.install
::ActionCable::Server::Worker.set_callback :work, :around, &wrapper
- ::ActionCable::Channel::Base.set_callback :subscribe, :around, &wrapper
- ::ActionCable::Channel::Base.set_callback :unsubscribe, :around, &wrapper
end
def self.wrapper
diff --git a/lib/gitlab/analytics/cycle_analytics/aggregated/records_fetcher.rb b/lib/gitlab/analytics/cycle_analytics/aggregated/records_fetcher.rb
index 07dc4c02ba8..2143497f084 100644
--- a/lib/gitlab/analytics/cycle_analytics/aggregated/records_fetcher.rb
+++ b/lib/gitlab/analytics/cycle_analytics/aggregated/records_fetcher.rb
@@ -94,10 +94,10 @@ module Gitlab
# rubocop: disable CodeReuse/ActiveRecord
def preload_associations(records)
- ActiveRecord::Associations::Preloader.new.preload(
- records,
- MAPPINGS.fetch(subject_class).fetch(:includes_for_query)
- )
+ ActiveRecord::Associations::Preloader.new(
+ records: records,
+ associations: MAPPINGS.fetch(subject_class).fetch(:includes_for_query)
+ ).call
records
end
diff --git a/lib/gitlab/analytics/cycle_analytics/aggregated/stage_query_helpers.rb b/lib/gitlab/analytics/cycle_analytics/aggregated/stage_query_helpers.rb
index b00925495f2..1d041e76277 100644
--- a/lib/gitlab/analytics/cycle_analytics/aggregated/stage_query_helpers.rb
+++ b/lib/gitlab/analytics/cycle_analytics/aggregated/stage_query_helpers.rb
@@ -32,7 +32,7 @@ module Gitlab
end
def duration_in_seconds(duration_expression = duration)
- Arel::Nodes::Extract.new(duration_expression, :epoch)
+ Arel::Nodes::NamedFunction.new('CAST', [Arel::Nodes::Extract.new(duration_expression, :epoch).as('double precision')])
end
end
end
diff --git a/lib/gitlab/analytics/cycle_analytics/average.rb b/lib/gitlab/analytics/cycle_analytics/average.rb
index 7140d31d536..4113e2e5d6a 100644
--- a/lib/gitlab/analytics/cycle_analytics/average.rb
+++ b/lib/gitlab/analytics/cycle_analytics/average.rb
@@ -41,7 +41,7 @@ module Gitlab
end
def average_in_seconds
- Arel::Nodes::Extract.new(average, :epoch)
+ Arel::Nodes::NamedFunction.new('CAST', [Arel::Nodes::Extract.new(average, :epoch).as('double precision')])
end
end
end
diff --git a/lib/gitlab/analytics/cycle_analytics/median.rb b/lib/gitlab/analytics/cycle_analytics/median.rb
index 5775d0324c6..0958cc39945 100644
--- a/lib/gitlab/analytics/cycle_analytics/median.rb
+++ b/lib/gitlab/analytics/cycle_analytics/median.rb
@@ -38,7 +38,8 @@ module Gitlab
end
def median_duration_in_seconds
- Arel::Nodes::Extract.new(percentile_cont, :epoch)
+ Arel::Nodes::NamedFunction.new('CAST',
+ [Arel::Nodes::Extract.new(percentile_cont, :epoch).as('double precision')])
end
end
end
diff --git a/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb b/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb
index 140c4a300ca..9deb5072112 100644
--- a/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb
+++ b/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb
@@ -67,10 +67,10 @@ module Gitlab
# rubocop: disable CodeReuse/ActiveRecord
def preload_associations(records)
# using preloader instead of includes to avoid AR generating a large column list
- ActiveRecord::Associations::Preloader.new.preload(
- records,
- MAPPINGS.fetch(subject_class).fetch(:includes_for_query)
- )
+ ActiveRecord::Associations::Preloader.new(
+ records: records,
+ associations: MAPPINGS.fetch(subject_class).fetch(:includes_for_query)
+ ).call
records
end
diff --git a/lib/gitlab/analytics/cycle_analytics/request_params.rb b/lib/gitlab/analytics/cycle_analytics/request_params.rb
index 2df3680db5f..2c4b0215307 100644
--- a/lib/gitlab/analytics/cycle_analytics/request_params.rb
+++ b/lib/gitlab/analytics/cycle_analytics/request_params.rb
@@ -38,13 +38,12 @@ module Gitlab
attribute :created_after, :datetime
attribute :created_before, :datetime
- attribute :group
+ attribute :namespace
attribute :current_user
attribute :value_stream
attribute :sort
attribute :direction
attribute :page
- attribute :project
attribute :stage_id
attribute :end_event_filter
@@ -66,10 +65,6 @@ module Gitlab
self.end_event_filter ||= Gitlab::Analytics::CycleAnalytics::BaseQueryBuilder::DEFAULT_END_EVENT_FILTER
end
- def project_ids
- Array(@project_ids)
- end
-
def to_data_collector_params
{
current_user: current_user,
@@ -86,12 +81,9 @@ module Gitlab
def to_data_attributes
{}.tap do |attrs|
- attrs[:aggregation] = aggregation_attributes if group
- attrs[:group] = group_data_attributes if group
attrs[:value_stream] = value_stream_data_attributes.to_json if value_stream
attrs[:created_after] = created_after.to_date.iso8601
attrs[:created_before] = created_before.to_date.iso8601
- attrs[:projects] = group_projects(project_ids) if group && project_ids.present?
attrs[:labels] = label_name.to_json if label_name.present?
attrs[:assignees] = assignee_username.to_json if assignee_username.present?
attrs[:author] = author_username if author_username.present?
@@ -99,35 +91,64 @@ module Gitlab
attrs[:sort] = sort if sort.present?
attrs[:direction] = direction if direction.present?
attrs[:stage] = stage_data_attributes.to_json if stage_id.present?
+ attrs[:namespace] = namespace_attributes
+ attrs[:enable_tasks_by_type_chart] = 'false'
+ attrs[:enable_customizable_stages] = 'false'
+ attrs[:enable_projects_filter] = 'false'
+ attrs[:default_stages] = Gitlab::Analytics::CycleAnalytics::DefaultStages.all.map do |stage_params|
+ ::Analytics::CycleAnalytics::StagePresenter.new(stage_params)
+ end.to_json
+
+ attrs.merge!(foss_project_level_params, resource_paths)
end
end
+ def project_ids
+ Array(@project_ids)
+ end
+
private
- def use_aggregated_backend?
- # for now it's only available on the group-level
- group.present?
- end
+ delegate :url_helpers, to: Gitlab::Routing
+
+ def foss_project_level_params
+ return {} unless project
- def aggregation_attributes
{
- enabled: aggregation.enabled.to_s,
- last_run_at: aggregation.last_incremental_run_at&.iso8601,
- next_run_at: aggregation.estimated_next_run_at&.iso8601
+ project_id: project.id,
+ group_path: project.group ? "groups/#{project.group&.full_path}" : nil,
+ request_path: url_helpers.project_cycle_analytics_path(project),
+ full_path: project.full_path
}
end
- def aggregation
- @aggregation ||= ::Analytics::CycleAnalytics::Aggregation.safe_create_for_namespace(group)
+ def resource_paths
+ helpers = ActionController::Base.helpers
+
+ {}.tap do |paths|
+ paths[:empty_state_svg_path] = helpers.image_path("illustrations/analytics/cycle-analytics-empty-chart.svg")
+ paths[:no_data_svg_path] = helpers.image_path("illustrations/analytics/cycle-analytics-empty-chart.svg")
+ paths[:no_access_svg_path] = helpers.image_path("illustrations/analytics/no-access.svg")
+
+ if project
+ paths[:milestones_path] = url_helpers.project_milestones_path(project, format: :json)
+ paths[:labels_path] = url_helpers.project_labels_path(project, format: :json)
+ end
+ end
end
- def group_data_attributes
+ # FOSS version doesn't use the aggregated VSA backend
+ def use_aggregated_backend?
+ false
+ end
+
+ def namespace_attributes
+ return {} unless project
+
{
- id: group.id,
- namespace_id: group.id,
- name: group.name,
- full_path: group.full_path,
- avatar_url: group.avatar_url
+ name: project.name,
+ full_path: project.full_path,
+ type: namespace.type
}
end
@@ -139,28 +160,6 @@ module Gitlab
}
end
- def group_projects(project_ids)
- GroupProjectsFinder.new(
- group: group,
- current_user: current_user,
- options: { include_subgroups: true },
- project_ids_relation: project_ids
- )
- .execute
- .with_route
- .map { |project| project_data_attributes(project) }
- .to_json
- end
-
- def project_data_attributes(project)
- {
- id: project.to_gid.to_s,
- name: project.name,
- path_with_namespace: project.path_with_namespace,
- avatar_url: project.avatar_url
- }
- end
-
def stage_data_attributes
return unless stage
@@ -196,10 +195,18 @@ module Gitlab
return unless value_stream
strong_memoize(:stage) do
- ::Analytics::CycleAnalytics::StageFinder.new(parent: project&.project_namespace || group, stage_id: stage_id).execute if stage_id
+ ::Analytics::CycleAnalytics::StageFinder.new(parent: namespace, stage_id: stage_id).execute if stage_id
+ end
+ end
+
+ def project
+ strong_memoize(:project) do
+ namespace.project if namespace.is_a?(Namespaces::ProjectNamespace)
end
end
end
end
end
end
+
+Gitlab::Analytics::CycleAnalytics::RequestParams.prepend_mod_with('Gitlab::Analytics::CycleAnalytics::RequestParams')
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start.rb
index 9b4cbc9090c..85dcc773e2b 100644
--- a/lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start.rb
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start.rb
@@ -6,7 +6,7 @@ module Gitlab
module StageEvents
class PlanStageStart < MetricsBasedStageEvent
def self.name
- s_("CycleAnalyticsEvent|Issue first associated with a milestone or issue first added to a board")
+ s_("CycleAnalyticsEvent|Issue first associated with a milestone or first added to a board")
end
def self.identifier
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb b/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb
index 5648984ecbb..3416a916e26 100644
--- a/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb
+++ b/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb
@@ -13,7 +13,9 @@ module Gitlab
end
def round_duration_to_seconds
- Arel::Nodes::NamedFunction.new('ROUND', [Arel::Nodes::Extract.new(duration, :epoch)])
+ Arel::Nodes::NamedFunction.new('ROUND', [
+ Arel::Nodes::NamedFunction.new('CAST', [Arel::Nodes::Extract.new(duration, :epoch).as('double precision')])
+ ])
end
def duration
diff --git a/lib/gitlab/app_logger.rb b/lib/gitlab/app_logger.rb
index a39e7f31886..decc6be3410 100644
--- a/lib/gitlab/app_logger.rb
+++ b/lib/gitlab/app_logger.rb
@@ -2,18 +2,12 @@
module Gitlab
class AppLogger < Gitlab::MultiDestinationLogger
- LOGGERS = [Gitlab::AppTextLogger, Gitlab::AppJsonLogger].freeze
-
def self.loggers
- if Gitlab::Utils.to_boolean(ENV.fetch('UNSTRUCTURED_RAILS_LOG', 'true'))
- LOGGERS
- else
- [Gitlab::AppJsonLogger]
- end
+ [Gitlab::AppJsonLogger]
end
def self.primary_logger
- Gitlab::AppTextLogger
+ Gitlab::AppJsonLogger
end
end
end
diff --git a/lib/gitlab/application_context.rb b/lib/gitlab/application_context.rb
index 06ce1dbdc77..0ea52b7b7c8 100644
--- a/lib/gitlab/application_context.rb
+++ b/lib/gitlab/application_context.rb
@@ -25,7 +25,8 @@ module Gitlab
:artifact_used_cdn,
:artifacts_dependencies_size,
:artifacts_dependencies_count,
- :root_caller_id
+ :root_caller_id,
+ :merge_action_status
].freeze
private_constant :KNOWN_KEYS
@@ -43,7 +44,8 @@ module Gitlab
Attribute.new(:artifact_used_cdn, Object),
Attribute.new(:artifacts_dependencies_size, Integer),
Attribute.new(:artifacts_dependencies_count, Integer),
- Attribute.new(:root_caller_id, String)
+ Attribute.new(:root_caller_id, String),
+ Attribute.new(:merge_action_status, String)
].freeze
def self.known_keys
@@ -97,6 +99,7 @@ module Gitlab
assign_hash_if_value(hash, :artifact_used_cdn)
assign_hash_if_value(hash, :artifacts_dependencies_size)
assign_hash_if_value(hash, :artifacts_dependencies_count)
+ assign_hash_if_value(hash, :merge_action_status)
hash[:user] = -> { username } if include_user?
hash[:user_id] = -> { user_id } if include_user?
diff --git a/lib/gitlab/application_rate_limiter.rb b/lib/gitlab/application_rate_limiter.rb
index 466538df56e..a8e74cbd7e6 100644
--- a/lib/gitlab/application_rate_limiter.rb
+++ b/lib/gitlab/application_rate_limiter.rb
@@ -30,6 +30,7 @@ module Gitlab
group_download_export: { threshold: -> { application_settings.group_download_export_limit }, interval: 1.minute },
group_import: { threshold: -> { application_settings.group_import_limit }, interval: 1.minute },
group_testing_hook: { threshold: 5, interval: 1.minute },
+ member_delete: { threshold: 60, interval: 1.minute },
profile_add_new_email: { threshold: 5, interval: 1.minute },
web_hook_calls: { interval: 1.minute },
web_hook_calls_mid: { interval: 1.minute },
@@ -55,8 +56,13 @@ module Gitlab
phone_verification_verify_code: { threshold: 10, interval: 10.minutes },
namespace_exists: { threshold: 20, interval: 1.minute },
fetch_google_ip_list: { threshold: 10, interval: 1.minute },
+ project_fork_sync: { threshold: 10, interval: 30.minutes },
+ ai_action: { threshold: 160, interval: 8.hours },
jobs_index: { threshold: 600, interval: 1.minute },
- bulk_import: { threshold: 6, interval: 1.minute }
+ bulk_import: { threshold: 6, interval: 1.minute },
+ projects_api_rate_limit_unauthenticated: {
+ threshold: -> { application_settings.projects_api_rate_limit_unauthenticated }, interval: 10.minutes
+ }
}.freeze
end
diff --git a/lib/gitlab/audit/auditor.rb b/lib/gitlab/audit/auditor.rb
index fddc1f830aa..e3d2b394404 100644
--- a/lib/gitlab/audit/auditor.rb
+++ b/lib/gitlab/audit/auditor.rb
@@ -5,6 +5,10 @@ module Gitlab
class Auditor
attr_reader :scope, :name
+ PERMITTED_TARGET_CLASSES = [
+ ::Operations::FeatureFlag
+ ].freeze
+
# Record audit events
#
# @param [Hash] context
@@ -113,7 +117,11 @@ module Gitlab
end
def audit_enabled?
- authentication_event?
+ authentication_event? || permitted_target?
+ end
+
+ def permitted_target?
+ @target.class.in? PERMITTED_TARGET_CLASSES
end
def authentication_event?
diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb
index 06bdb2c1ddc..9268fdd8519 100644
--- a/lib/gitlab/auth.rb
+++ b/lib/gitlab/auth.rb
@@ -3,7 +3,7 @@
module Gitlab
module Auth
MissingPersonalAccessTokenError = Class.new(StandardError)
- IpBlacklisted = Class.new(StandardError)
+ IpBlocked = Class.new(StandardError)
# Scopes used for GitLab API access
API_SCOPE = :api
@@ -29,6 +29,12 @@ module Gitlab
WRITE_REGISTRY_SCOPE = :write_registry
REGISTRY_SCOPES = [READ_REGISTRY_SCOPE, WRITE_REGISTRY_SCOPE].freeze
+ # Scopes used for GitLab Observability access which is outside of the GitLab app itself.
+ # Hence the lack of ability mapping in `abilities_for_scopes`.
+ READ_OBSERVABILITY_SCOPE = :read_observability
+ WRITE_OBSERVABILITY_SCOPE = :write_observability
+ OBSERVABILITY_SCOPES = [READ_OBSERVABILITY_SCOPE, WRITE_OBSERVABILITY_SCOPE].freeze
+
# Scopes used for GitLab as admin
SUDO_SCOPE = :sudo
ADMIN_MODE_SCOPE = :admin_mode
@@ -51,7 +57,7 @@ module Gitlab
rate_limiter = Gitlab::Auth::IpRateLimiter.new(ip)
- raise IpBlacklisted if !skip_rate_limit?(login: login) && rate_limiter.banned?
+ raise IpBlocked if !skip_rate_limit?(login: login) && rate_limiter.banned?
# `user_with_password_for_git` should be the last check
# because it's the most expensive, especially when LDAP
@@ -364,14 +370,8 @@ module Gitlab
]
end
- def available_scopes_for(current_user)
- scopes = non_admin_available_scopes
-
- if current_user.admin? # rubocop: disable Cop/UserAdmin
- scopes += Feature.enabled?(:admin_mode_for_api) ? ADMIN_SCOPES : [SUDO_SCOPE]
- end
-
- scopes
+ def available_scopes_for(resource)
+ available_scopes_for_resource(resource) - unavailable_scopes_for_resource(resource)
end
def all_available_scopes
@@ -390,13 +390,40 @@ module Gitlab
end
def resource_bot_scopes
- Gitlab::Auth::API_SCOPES + Gitlab::Auth::REPOSITORY_SCOPES + Gitlab::Auth.registry_scopes - [:read_user]
+ non_admin_available_scopes - [READ_USER_SCOPE]
end
private
+ def available_scopes_for_resource(resource)
+ case resource
+ when User
+ scopes = non_admin_available_scopes
+
+ if resource.admin? # rubocop: disable Cop/UserAdmin
+ scopes += Feature.enabled?(:admin_mode_for_api) ? ADMIN_SCOPES : [SUDO_SCOPE]
+ end
+
+ scopes
+ when Project, Group
+ resource_bot_scopes
+ else
+ []
+ end
+ end
+
+ def unavailable_scopes_for_resource(resource)
+ unavailable_observability_scopes_for_resource(resource)
+ end
+
+ def unavailable_observability_scopes_for_resource(resource)
+ return [] if resource.is_a?(Group) && Gitlab::Observability.enabled?(resource)
+
+ OBSERVABILITY_SCOPES
+ end
+
def non_admin_available_scopes
- API_SCOPES + REPOSITORY_SCOPES + registry_scopes
+ API_SCOPES + REPOSITORY_SCOPES + registry_scopes + OBSERVABILITY_SCOPES
end
def find_build_by_token(token)
diff --git a/lib/gitlab/auth/auth_finders.rb b/lib/gitlab/auth/auth_finders.rb
index c69462b12de..4a610b26290 100644
--- a/lib/gitlab/auth/auth_finders.rb
+++ b/lib/gitlab/auth/auth_finders.rb
@@ -225,6 +225,12 @@ module Gitlab
def access_token
strong_memoize(:access_token) do
+ # Kubernetes API OAuth header is not OauthAccessToken or PersonalAccessToken
+ # and should be ignored by this method. When the kubernetes API uses a different
+ # header, we can remove this guard
+ # https://gitlab.com/gitlab-org/gitlab/-/issues/406582
+ next if current_request.path.starts_with? "/api/v4/internal/kubernetes/"
+
if try(:namespace_inheritable, :authentication)
access_token_from_namespace_inheritable
else
diff --git a/lib/gitlab/auth/ldap/config.rb b/lib/gitlab/auth/ldap/config.rb
index 6c99b505797..30896637eff 100644
--- a/lib/gitlab/auth/ldap/config.rb
+++ b/lib/gitlab/auth/ldap/config.rb
@@ -184,6 +184,10 @@ module Gitlab
options['lowercase_usernames']
end
+ def sync_name
+ options['sync_name']
+ end
+
def name_proc
if allow_username_or_email_login
proc { |name| name.gsub(/@.*\z/, '') }
diff --git a/lib/gitlab/auth/o_auth/auth_hash.rb b/lib/gitlab/auth/o_auth/auth_hash.rb
index 82a5aad360c..d1eede65f0c 100644
--- a/lib/gitlab/auth/o_auth/auth_hash.rb
+++ b/lib/gitlab/auth/o_auth/auth_hash.rb
@@ -76,7 +76,13 @@ module Gitlab
end
def get_from_auth_hash_or_info(key)
- coerce_utf8(auth_hash[key]) || get_info(key)
+ if auth_hash.key?(key)
+ coerce_utf8(auth_hash[key])
+ elsif auth_hash.key?(:extra) && auth_hash.extra.key?(:raw_info) && !auth_hash.extra.raw_info[key].nil?
+ coerce_utf8(auth_hash.extra.raw_info[key])
+ else
+ get_info(key)
+ end
end
# Allow for configuring a custom username claim per provider from
diff --git a/lib/gitlab/auth/o_auth/session.rb b/lib/gitlab/auth/o_auth/session.rb
deleted file mode 100644
index 4925b107042..00000000000
--- a/lib/gitlab/auth/o_auth/session.rb
+++ /dev/null
@@ -1,23 +0,0 @@
-# frozen_string_literal: true
-
-# :nocov:
-module Gitlab
- module Auth
- module OAuth
- module Session
- def self.create(provider, ticket)
- Rails.cache.write("gitlab:#{provider}:#{ticket}", ticket, expires_in: Gitlab.config.omniauth.cas3.session_duration)
- end
-
- def self.destroy(provider, ticket)
- Rails.cache.delete("gitlab:#{provider}:#{ticket}")
- end
-
- def self.valid?(provider, ticket)
- Rails.cache.read("gitlab:#{provider}:#{ticket}").present?
- end
- end
- end
- end
-end
-# :nocov:
diff --git a/lib/gitlab/auth/o_auth/user.rb b/lib/gitlab/auth/o_auth/user.rb
index 01e126ec2f5..3981594478d 100644
--- a/lib/gitlab/auth/o_auth/user.rb
+++ b/lib/gitlab/auth/o_auth/user.rb
@@ -233,7 +233,7 @@ module Gitlab
email ||= auth_hash.email
valid_username = ::Namespace.clean_path(username)
- valid_username = Uniquify.new.string(valid_username) { |s| !NamespacePathValidator.valid_path?(s) }
+ valid_username = Gitlab::Utils::Uniquify.new.string(valid_username) { |s| !NamespacePathValidator.valid_path?(s) }
{
name: name.strip.presence || valid_username,
@@ -258,7 +258,7 @@ module Gitlab
metadata = gl_user.build_user_synced_attributes_metadata
if sync_profile_from_provider?
- UserSyncedAttributesMetadata.syncable_attributes.each do |key|
+ UserSyncedAttributesMetadata.syncable_attributes(auth_hash.provider).each do |key|
if auth_hash.has_attribute?(key) && gl_user.sync_attribute?(key)
gl_user.public_send("#{key}=".to_sym, auth_hash.public_send(key)) # rubocop:disable GitlabSecurity/PublicSend
metadata.set_attribute_synced(key, true)
diff --git a/lib/gitlab/auth/otp/duo_auth.rb b/lib/gitlab/auth/otp/duo_auth.rb
new file mode 100644
index 00000000000..eeae04bc08b
--- /dev/null
+++ b/lib/gitlab/auth/otp/duo_auth.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Auth
+ module Otp
+ module DuoAuth
+ def duo_auth_enabled?(_user)
+ ::Gitlab.config.duo_auth.enabled
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/auth/otp/strategies/duo_auth/manual_otp.rb b/lib/gitlab/auth/otp/strategies/duo_auth/manual_otp.rb
new file mode 100644
index 00000000000..57bc88de175
--- /dev/null
+++ b/lib/gitlab/auth/otp/strategies/duo_auth/manual_otp.rb
@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Auth
+ module Otp
+ module Strategies
+ module DuoAuth
+ class ManualOtp < Base
+ include Gitlab::Utils::StrongMemoize
+
+ def validate(otp_code)
+ params = { username: user.username, factor: "passcode", passcode: otp_code.to_i }
+ response = duo_client.request('POST', "/auth/v2/auth", params)
+ approve_or_deny(parse_response(response))
+ rescue StandardError => e
+ Gitlab::AppLogger.error(e)
+ error(e.message)
+ end
+
+ private
+
+ def duo_client
+ DuoApi.new(::Gitlab.config.duo_auth.integration_key,
+ ::Gitlab.config.duo_auth.secret_key,
+ ::Gitlab.config.duo_auth.hostname)
+ end
+ strong_memoize_attr :duo_client
+
+ def parse_response(response)
+ Gitlab::Json.parse(response.body)
+ end
+
+ def approve_or_deny(parsed_response)
+ result_key = parsed_response.dig('response', 'result')
+ if result_key.to_s == "allow"
+ success
+ else
+ error(message: parsed_response.dig('response', 'status_msg').to_s)
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/auth/u2f_webauthn_converter.rb b/lib/gitlab/auth/u2f_webauthn_converter.rb
deleted file mode 100644
index 20b5d2ddc88..00000000000
--- a/lib/gitlab/auth/u2f_webauthn_converter.rb
+++ /dev/null
@@ -1,40 +0,0 @@
-# frozen_string_literal: true
-
-require 'webauthn/u2f_migrator'
-
-module Gitlab
- module Auth
- class U2fWebauthnConverter
- def initialize(u2f_registration)
- @u2f_registration = u2f_registration
- end
-
- def convert
- now = Time.current
-
- converted_credential = WebAuthn::U2fMigrator.new(
- app_id: Gitlab.config.gitlab.url,
- certificate: u2f_registration.certificate,
- key_handle: u2f_registration.key_handle,
- public_key: u2f_registration.public_key,
- counter: u2f_registration.counter
- ).credential
-
- {
- credential_xid: Base64.strict_encode64(converted_credential.id),
- public_key: Base64.strict_encode64(converted_credential.public_key),
- counter: u2f_registration.counter || 0,
- name: u2f_registration.name || '',
- user_id: u2f_registration.user_id,
- u2f_registration_id: u2f_registration.id,
- created_at: now,
- updated_at: now
- }
- end
-
- private
-
- attr_reader :u2f_registration
- end
- end
-end
diff --git a/lib/gitlab/auth/user_access_denied_reason.rb b/lib/gitlab/auth/user_access_denied_reason.rb
index 322dfa74d09..3025960a8ab 100644
--- a/lib/gitlab/auth/user_access_denied_reason.rb
+++ b/lib/gitlab/auth/user_access_denied_reason.rb
@@ -29,7 +29,8 @@ module Gitlab
"Your password expired. "\
"Please access GitLab from a web browser to update your password."
else
- "Your account has been blocked."
+ "Your request has been rejected for an unknown reason."\
+ "Please contact your GitLab administrator and/or GitLab Support."
end
end
diff --git a/lib/gitlab/avatar_cache.rb b/lib/gitlab/avatar_cache.rb
index 30c8e089061..ed00a279299 100644
--- a/lib/gitlab/avatar_cache.rb
+++ b/lib/gitlab/avatar_cache.rb
@@ -65,7 +65,13 @@ module Gitlab
keys = emails.map { |email| email_key(email) }
Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do
- redis.unlink(*keys)
+ if ::Feature.enabled?(:use_pipeline_over_multikey)
+ redis.pipelined do |pipeline|
+ keys.each { |key| pipeline.unlink(key) }
+ end.sum
+ else
+ redis.unlink(*keys)
+ end
end
end
end
diff --git a/lib/gitlab/background_migration/backfill_admin_mode_scope_for_personal_access_tokens.rb b/lib/gitlab/background_migration/backfill_admin_mode_scope_for_personal_access_tokens.rb
index 82e607ac7a7..2127ce5975d 100644
--- a/lib/gitlab/background_migration/backfill_admin_mode_scope_for_personal_access_tokens.rb
+++ b/lib/gitlab/background_migration/backfill_admin_mode_scope_for_personal_access_tokens.rb
@@ -12,14 +12,18 @@ module Gitlab
end
operation_name :update_all
- feature_category :authentication_and_authorization
+ feature_category :system_access
ADMIN_MODE_SCOPE = ['admin_mode'].freeze
def perform
each_sub_batch do |sub_batch|
sub_batch.each do |token|
- token.update!(scopes: (YAML.safe_load(token.scopes) + ADMIN_MODE_SCOPE).uniq.to_yaml)
+ existing_scopes = YAML.safe_load(token.scopes, permitted_classes: [Symbol])
+ # making sure scopes are not mixed symbols and strings
+ stringified_scopes = existing_scopes.map(&:to_s)
+
+ token.update!(scopes: (stringified_scopes + ADMIN_MODE_SCOPE).uniq.to_yaml)
end
end
end
diff --git a/lib/gitlab/background_migration/backfill_compliance_violations.rb b/lib/gitlab/background_migration/backfill_compliance_violations.rb
new file mode 100644
index 00000000000..131b4a05e41
--- /dev/null
+++ b/lib/gitlab/background_migration/backfill_compliance_violations.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ # rubocop: disable Style/Documentation
+ class BackfillComplianceViolations < Gitlab::BackgroundMigration::BatchedMigrationJob
+ feature_category :compliance_management
+
+ def perform
+ # no-op. The logic is defined in EE module.
+ end
+ end
+ # rubocop: enable Style/Documentation
+ end
+end
+
+::Gitlab::BackgroundMigration::BackfillComplianceViolations.prepend_mod
diff --git a/lib/gitlab/background_migration/backfill_design_management_repositories.rb b/lib/gitlab/background_migration/backfill_design_management_repositories.rb
new file mode 100644
index 00000000000..fe57767a693
--- /dev/null
+++ b/lib/gitlab/background_migration/backfill_design_management_repositories.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ # Backfill design_management_repositories table for a range of projects
+ class BackfillDesignManagementRepositories < BatchedMigrationJob
+ operation_name :backfill_design_management_repositories
+ feature_category :geo_replication
+
+ def perform
+ each_sub_batch do |sub_batch|
+ backfill_design_management_repositories(sub_batch)
+ end
+ end
+
+ def backfill_design_management_repositories(relation)
+ connection.execute(
+ <<~SQL
+ INSERT INTO design_management_repositories (project_id, created_at, updated_at)
+ SELECT projects.id, now(), now()
+ FROM projects
+ WHERE projects.id IN(#{relation.select(:id).to_sql})
+ ON CONFLICT (project_id) DO NOTHING;
+ SQL
+ )
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/background_migration/backfill_integrations_enable_ssl_verification.rb b/lib/gitlab/background_migration/backfill_integrations_enable_ssl_verification.rb
index de52629522b..878f89a8b3d 100644
--- a/lib/gitlab/background_migration/backfill_integrations_enable_ssl_verification.rb
+++ b/lib/gitlab/background_migration/backfill_integrations_enable_ssl_verification.rb
@@ -40,13 +40,13 @@ module Gitlab
scope :affected, -> { where(type_new: INTEGRATIONS.keys).where.not(encrypted_properties: nil) }
attr_encrypted :properties,
- mode: :per_attribute_iv,
- key: Settings.attr_encrypted_db_key_base_32,
- algorithm: 'aes-256-gcm',
- marshal: true,
- marshaler: ::Gitlab::Json,
- encode: false,
- encode_iv: false
+ mode: :per_attribute_iv,
+ key: Settings.attr_encrypted_db_key_base_32,
+ algorithm: 'aes-256-gcm',
+ marshal: true,
+ marshaler: ::Gitlab::Json,
+ encode: false,
+ encode_iv: false
# Handle assignment of props with symbol keys.
# To do this correctly, we need to call the method generated by attr_encrypted.
diff --git a/lib/gitlab/background_migration/backfill_namespace_ldap_settings.rb b/lib/gitlab/background_migration/backfill_namespace_ldap_settings.rb
new file mode 100644
index 00000000000..1a5ad1c14a6
--- /dev/null
+++ b/lib/gitlab/background_migration/backfill_namespace_ldap_settings.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ # Back-fill container_registry_size for project_statistics
+ class BackfillNamespaceLdapSettings < Gitlab::BackgroundMigration::BatchedMigrationJob
+ operation_name :backfill_namespace_ldap_settings
+ feature_category :system_access
+
+ def perform
+ # no-op in FOSS
+ end
+ end
+ end
+end
+
+Gitlab::BackgroundMigration::BackfillNamespaceLdapSettings.prepend_mod
diff --git a/lib/gitlab/background_migration/backfill_namespace_traversal_ids_children.rb b/lib/gitlab/background_migration/backfill_namespace_traversal_ids_children.rb
deleted file mode 100644
index 3b8a452b855..00000000000
--- a/lib/gitlab/background_migration/backfill_namespace_traversal_ids_children.rb
+++ /dev/null
@@ -1,76 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module BackgroundMigration
- # A job to set namespaces.traversal_ids in sub-batches, of all namespaces with
- # a parent and not already set.
- # rubocop:disable Style/Documentation
- class BackfillNamespaceTraversalIdsChildren
- class Namespace < ActiveRecord::Base
- include ::EachBatch
-
- self.table_name = 'namespaces'
-
- scope :base_query, -> { where.not(parent_id: nil) }
- end
-
- PAUSE_SECONDS = 0.1
-
- def perform(start_id, end_id, sub_batch_size)
- batch_query = Namespace.base_query.where(id: start_id..end_id)
- batch_query.each_batch(of: sub_batch_size) do |sub_batch|
- first, last = sub_batch.pick(Arel.sql('min(id), max(id)'))
- ranged_query = Namespace.unscoped.base_query.where(id: first..last)
-
- update_sql = <<~SQL
- UPDATE namespaces
- SET traversal_ids = calculated_ids.traversal_ids
- FROM #{calculated_traversal_ids(ranged_query)} calculated_ids
- WHERE namespaces.id = calculated_ids.id
- AND namespaces.traversal_ids = '{}'
- SQL
- ApplicationRecord.connection.execute(update_sql)
-
- sleep PAUSE_SECONDS
- end
-
- # We have to add all arguments when marking a job as succeeded as they
- # are all used to track the job by `queue_background_migration_jobs_by_range_at_intervals`
- mark_job_as_succeeded(start_id, end_id, sub_batch_size)
- end
-
- private
-
- # Calculate the ancestor path for a given set of namespaces.
- def calculated_traversal_ids(batch)
- <<~SQL
- (
- WITH RECURSIVE cte(source_id, namespace_id, parent_id, height) AS (
- (
- SELECT batch.id, batch.id, batch.parent_id, 1
- FROM (#{batch.to_sql}) AS batch
- )
- UNION ALL
- (
- SELECT cte.source_id, n.id, n.parent_id, cte.height+1
- FROM namespaces n, cte
- WHERE n.id = cte.parent_id
- )
- )
- SELECT flat_hierarchy.source_id as id,
- array_agg(flat_hierarchy.namespace_id ORDER BY flat_hierarchy.height DESC) as traversal_ids
- FROM (SELECT * FROM cte FOR UPDATE) flat_hierarchy
- GROUP BY flat_hierarchy.source_id
- )
- SQL
- end
-
- def mark_job_as_succeeded(*arguments)
- Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded(
- 'BackfillNamespaceTraversalIdsChildren',
- arguments
- )
- end
- end
- end
-end
diff --git a/lib/gitlab/background_migration/backfill_namespace_traversal_ids_roots.rb b/lib/gitlab/background_migration/backfill_namespace_traversal_ids_roots.rb
deleted file mode 100644
index c69289fb91f..00000000000
--- a/lib/gitlab/background_migration/backfill_namespace_traversal_ids_roots.rb
+++ /dev/null
@@ -1,51 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module BackgroundMigration
- # A job to set namespaces.traversal_ids in sub-batches, of all namespaces
- # without a parent and not already set.
- # rubocop:disable Style/Documentation
- class BackfillNamespaceTraversalIdsRoots
- class Namespace < ActiveRecord::Base
- include ::EachBatch
-
- self.table_name = 'namespaces'
-
- scope :base_query, -> { where(parent_id: nil) }
- end
-
- PAUSE_SECONDS = 0.1
-
- def perform(start_id, end_id, sub_batch_size)
- ranged_query = Namespace.base_query
- .where(id: start_id..end_id)
- .where("traversal_ids = '{}'")
-
- ranged_query.each_batch(of: sub_batch_size) do |sub_batch|
- first, last = sub_batch.pick(Arel.sql('min(id), max(id)'))
-
- # The query need to be reconstructed because .each_batch modifies the default scope
- # See: https://gitlab.com/gitlab-org/gitlab/-/issues/330510
- Namespace.unscoped
- .base_query
- .where(id: first..last)
- .where("traversal_ids = '{}'")
- .update_all('traversal_ids = ARRAY[id]')
-
- sleep PAUSE_SECONDS
- end
-
- mark_job_as_succeeded(start_id, end_id, sub_batch_size)
- end
-
- private
-
- def mark_job_as_succeeded(*arguments)
- Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded(
- 'BackfillNamespaceTraversalIdsRoots',
- arguments
- )
- end
- end
- end
-end
diff --git a/lib/gitlab/background_migration/backfill_partitioned_table.rb b/lib/gitlab/background_migration/backfill_partitioned_table.rb
new file mode 100644
index 00000000000..6479d40a930
--- /dev/null
+++ b/lib/gitlab/background_migration/backfill_partitioned_table.rb
@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ # Background migration to generically copy data from the given table into its corresponding partitioned table
+ class BackfillPartitionedTable < BatchedMigrationJob
+ operation_name :upsert_partitioned_table
+ feature_category :database
+ job_arguments :partitioned_table
+
+ def perform
+ validate_paritition_table!
+
+ bulk_copy = Gitlab::Database::PartitioningMigrationHelpers::BulkCopy.new(
+ batch_table,
+ partitioned_table,
+ batch_column,
+ connection: connection
+ )
+
+ each_sub_batch do |relation|
+ sub_start_id, sub_stop_id = relation.pick(Arel.sql("MIN(#{batch_column}), MAX(#{batch_column})"))
+ bulk_copy.copy_between(sub_start_id, sub_stop_id)
+ end
+ end
+
+ private
+
+ def validate_paritition_table!
+ unless connection.table_exists?(partitioned_table)
+ raise "exiting backfill migration because partitioned table #{partitioned_table} does not exist. " \
+ "This could be due to rollback of the migration which created the partitioned table."
+ end
+
+ # rubocop: disable Style/GuardClause
+ unless Gitlab::Database::PostgresPartitionedTable.find_by_name_in_current_schema(partitioned_table).present?
+ raise "exiting backfill migration because the given destination table is not partitioned."
+ end
+ # rubocop: enable Style/GuardClause
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/background_migration/backfill_prepared_at_merge_requests.rb b/lib/gitlab/background_migration/backfill_prepared_at_merge_requests.rb
new file mode 100644
index 00000000000..9bf503bd6e7
--- /dev/null
+++ b/lib/gitlab/background_migration/backfill_prepared_at_merge_requests.rb
@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ # Backfill prepared_at for an array of merge requests
+ class BackfillPreparedAtMergeRequests < ::Gitlab::BackgroundMigration::BatchedMigrationJob
+ scope_to ->(relation) { relation }
+ operation_name :update_all
+ feature_category :code_review_workflow
+
+ def perform
+ each_sub_batch do |sub_batch|
+ sub_batch.where(prepared_at: nil).where.not(merge_status: 'preparing').update_all('prepared_at = created_at')
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/background_migration/backfill_project_wiki_repositories.rb b/lib/gitlab/background_migration/backfill_project_wiki_repositories.rb
new file mode 100644
index 00000000000..8d6df905f15
--- /dev/null
+++ b/lib/gitlab/background_migration/backfill_project_wiki_repositories.rb
@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ # Backfill project_wiki_repositories table for a range of projects
+ class BackfillProjectWikiRepositories < BatchedMigrationJob
+ operation_name :backfill_project_wiki_repositories
+ feature_category :geo_replication
+
+ scope_to ->(relation) do
+ relation
+ .joins('LEFT OUTER JOIN project_wiki_repositories ON project_wiki_repositories.project_id = projects.id')
+ .where(project_wiki_repositories: { project_id: nil })
+ end
+
+ def perform
+ each_sub_batch do |sub_batch|
+ backfill_project_wiki_repositories(sub_batch)
+ end
+ end
+
+ def backfill_project_wiki_repositories(relation)
+ connection.execute(
+ <<~SQL
+ INSERT INTO project_wiki_repositories (project_id, created_at, updated_at)
+ SELECT projects.id, now(), now()
+ FROM projects
+ WHERE projects.id IN(#{relation.select(:id).to_sql})
+ ON CONFLICT (project_id) DO NOTHING;
+ SQL
+ )
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/background_migration/backfill_upvotes_count_on_issues.rb b/lib/gitlab/background_migration/backfill_upvotes_count_on_issues.rb
deleted file mode 100644
index 3bf6bf993dd..00000000000
--- a/lib/gitlab/background_migration/backfill_upvotes_count_on_issues.rb
+++ /dev/null
@@ -1,40 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module BackgroundMigration
- # Class that will populate the upvotes_count field
- # for each issue
- class BackfillUpvotesCountOnIssues
- BATCH_SIZE = 1_000
-
- def perform(start_id, stop_id)
- (start_id..stop_id).step(BATCH_SIZE).each do |offset|
- update_issue_upvotes_count(offset, offset + BATCH_SIZE)
- end
- end
-
- private
-
- def execute(sql)
- @connection ||= ApplicationRecord.connection
- @connection.execute(sql)
- end
-
- def update_issue_upvotes_count(batch_start, batch_stop)
- execute(<<~SQL)
- UPDATE issues
- SET upvotes_count = sub_q.count_all
- FROM (
- SELECT COUNT(*) AS count_all, e.awardable_id AS issue_id
- FROM award_emoji AS e
- WHERE e.name = 'thumbsup' AND
- e.awardable_type = 'Issue' AND
- e.awardable_id BETWEEN #{batch_start} AND #{batch_stop}
- GROUP BY issue_id
- ) AS sub_q
- WHERE sub_q.issue_id = issues.id;
- SQL
- end
- end
- end
-end
diff --git a/lib/gitlab/background_migration/backfill_user_namespace.rb b/lib/gitlab/background_migration/backfill_user_namespace.rb
deleted file mode 100644
index df6b1f083c3..00000000000
--- a/lib/gitlab/background_migration/backfill_user_namespace.rb
+++ /dev/null
@@ -1,38 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module BackgroundMigration
- # Backfills the `namespaces.type` column, replacing any
- # instances of `NULL` with `User`
- class BackfillUserNamespace
- include Gitlab::Database::DynamicModelHelpers
-
- def perform(start_id, end_id, batch_table, batch_column, sub_batch_size, pause_ms)
- parent_batch_relation = relation_scoped_to_range(batch_table, batch_column, start_id, end_id)
- parent_batch_relation.each_batch(column: batch_column, of: sub_batch_size, order_hint: :type) do |sub_batch|
- batch_metrics.time_operation(:update_all) do
- sub_batch.update_all(type: 'User')
- end
- pause_ms = 0 if pause_ms < 0
- sleep(pause_ms * 0.001)
- end
- end
-
- def batch_metrics
- @batch_metrics ||= Gitlab::Database::BackgroundMigration::BatchMetrics.new
- end
-
- private
-
- def connection
- ApplicationRecord.connection
- end
-
- def relation_scoped_to_range(source_table, source_key_column, start_id, stop_id)
- define_batchable_model(source_table, connection: connection)
- .where(source_key_column => start_id..stop_id)
- .where(type: nil)
- end
- end
- end
-end
diff --git a/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues.rb b/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues.rb
index fc0d0ce3a57..8e2e588e0cd 100644
--- a/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues.rb
+++ b/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues.rb
@@ -11,7 +11,7 @@ module Gitlab
class MigrationIssue < ApplicationRecord
self.table_name = 'issues'
- scope :base_query, ->(base_type) { where(work_item_type_id: nil, issue_type: base_type) }
+ scope :base_query, ->(base_type) { where(issue_type: base_type) }
end
MAX_UPDATE_RETRIES = 3
@@ -24,9 +24,7 @@ module Gitlab
operation_name :update_all
def perform
- each_sub_batch(
- batching_scope: -> (relation) { relation.where(work_item_type_id: nil) }
- ) do |sub_batch|
+ each_sub_batch do |sub_batch|
first, last = sub_batch.pick(Arel.sql('min(id), max(id)'))
# The query need to be reconstructed because .each_batch modifies the default scope
diff --git a/lib/gitlab/background_migration/batched_migration_job.rb b/lib/gitlab/background_migration/batched_migration_job.rb
index 4039a79cfa7..952e6d01f1a 100644
--- a/lib/gitlab/background_migration/batched_migration_job.rb
+++ b/lib/gitlab/background_migration/batched_migration_job.rb
@@ -7,6 +7,8 @@ module Gitlab
#
# Job arguments needed must be defined explicitly,
# see https://docs.gitlab.com/ee/development/database/batched_background_migrations.html#job-arguments.
+ # rubocop:disable Metrics/ClassLength
+ # rubocop:disable Metrics/ParameterLists
class BatchedMigrationJob
include Gitlab::Database::DynamicModelHelpers
include Gitlab::ClassAttributes
@@ -60,7 +62,8 @@ module Gitlab
end
def initialize(
- start_id:, end_id:, batch_table:, batch_column:, sub_batch_size:, pause_ms:, job_arguments: [], connection:
+ start_id:, end_id:, batch_table:, batch_column:, sub_batch_size:, pause_ms:, job_arguments: [], connection:,
+ sub_batch_exception: nil
)
@start_id = start_id
@@ -71,6 +74,7 @@ module Gitlab
@pause_ms = pause_ms
@job_arguments = job_arguments
@connection = connection
+ @sub_batch_exception = sub_batch_exception
end
def filter_batch(relation)
@@ -87,7 +91,8 @@ module Gitlab
private
- attr_reader :start_id, :end_id, :batch_table, :batch_column, :sub_batch_size, :pause_ms, :connection
+ attr_reader :start_id, :end_id, :batch_table, :batch_column, :sub_batch_size,
+ :pause_ms, :connection, :sub_batch_exception
def each_sub_batch(batching_arguments: {}, batching_scope: nil)
all_batching_arguments = { column: batch_column, of: sub_batch_size }.merge(batching_arguments)
@@ -98,6 +103,10 @@ module Gitlab
sub_batch_relation.each_batch(**all_batching_arguments) do |relation|
batch_metrics.instrument_operation(operation_name) do
yield relation
+ rescue *Gitlab::Database::BackgroundMigration::BatchedJob::TIMEOUT_EXCEPTIONS => exception
+ exception_class = sub_batch_exception || exception.class
+
+ raise exception_class, exception
end
sleep([pause_ms, 0].max * 0.001)
@@ -137,3 +146,5 @@ module Gitlab
end
end
end
+# rubocop:enable Metrics/ClassLength
+# rubocop:enable Metrics/ParameterLists
diff --git a/lib/gitlab/background_migration/cleanup_orphaned_lfs_objects_projects.rb b/lib/gitlab/background_migration/cleanup_orphaned_lfs_objects_projects.rb
deleted file mode 100644
index 4da120769a0..00000000000
--- a/lib/gitlab/background_migration/cleanup_orphaned_lfs_objects_projects.rb
+++ /dev/null
@@ -1,78 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module BackgroundMigration
- # The migration is used to cleanup orphaned lfs_objects_projects in order to
- # introduce valid foreign keys to this table
- class CleanupOrphanedLfsObjectsProjects
- # A model to access lfs_objects_projects table in migrations
- class LfsObjectsProject < ActiveRecord::Base
- self.table_name = 'lfs_objects_projects'
-
- include ::EachBatch
-
- belongs_to :lfs_object
- belongs_to :project
- end
-
- # A model to access lfs_objects table in migrations
- class LfsObject < ActiveRecord::Base
- self.table_name = 'lfs_objects'
- end
-
- # A model to access projects table in migrations
- class Project < ActiveRecord::Base
- self.table_name = 'projects'
- end
-
- SUB_BATCH_SIZE = 5000
- CLEAR_CACHE_DELAY = 1.minute
-
- def perform(start_id, end_id)
- cleanup_lfs_objects_projects_without_lfs_object(start_id, end_id)
- cleanup_lfs_objects_projects_without_project(start_id, end_id)
- end
-
- private
-
- def cleanup_lfs_objects_projects_without_lfs_object(start_id, end_id)
- each_record_without_association(start_id, end_id, :lfs_object, :lfs_objects) do |lfs_objects_projects_without_lfs_objects|
- projects = Project.where(id: lfs_objects_projects_without_lfs_objects.select(:project_id))
-
- if projects.present?
- ProjectCacheWorker.bulk_perform_in_with_contexts(
- CLEAR_CACHE_DELAY,
- projects,
- arguments_proc: ->(project) { [project.id, [], [:lfs_objects_size]] },
- context_proc: ->(project) { { project: project } }
- )
- end
-
- lfs_objects_projects_without_lfs_objects.delete_all
- end
- end
-
- def cleanup_lfs_objects_projects_without_project(start_id, end_id)
- each_record_without_association(start_id, end_id, :project, :projects) do |lfs_objects_projects_without_projects|
- lfs_objects_projects_without_projects.delete_all
- end
- end
-
- def each_record_without_association(start_id, end_id, association, table_name)
- batch = LfsObjectsProject.where(id: start_id..end_id)
-
- batch.each_batch(of: SUB_BATCH_SIZE) do |sub_batch|
- first, last = sub_batch.pick(Arel.sql('min(lfs_objects_projects.id), max(lfs_objects_projects.id)'))
-
- lfs_objects_without_association =
- LfsObjectsProject
- .unscoped
- .left_outer_joins(association)
- .where(id: (first..last), table_name => { id: nil })
-
- yield lfs_objects_without_association
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/background_migration/cleanup_personal_access_tokens_with_nil_expires_at.rb b/lib/gitlab/background_migration/cleanup_personal_access_tokens_with_nil_expires_at.rb
new file mode 100644
index 00000000000..e8ee2a4c251
--- /dev/null
+++ b/lib/gitlab/background_migration/cleanup_personal_access_tokens_with_nil_expires_at.rb
@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ # Clean up personal access tokens with expires_at value is nil
+ # and set the value to new default 365 days
+ class CleanupPersonalAccessTokensWithNilExpiresAt < BatchedMigrationJob
+ feature_category :system_access
+
+ EXPIRES_AT_DEFAULT = 365.days.from_now
+
+ scope_to ->(relation) { relation.where(expires_at: nil) }
+ operation_name :update_all
+
+ def perform
+ each_sub_batch do |sub_batch|
+ sub_batch.update_all(expires_at: EXPIRES_AT_DEFAULT)
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/background_migration/create_vulnerability_links.rb b/lib/gitlab/background_migration/create_vulnerability_links.rb
new file mode 100644
index 00000000000..bbc71dfb392
--- /dev/null
+++ b/lib/gitlab/background_migration/create_vulnerability_links.rb
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+# rubocop:disable Style/Documentation
+module Gitlab
+ module BackgroundMigration
+ class CreateVulnerabilityLinks < BatchedMigrationJob
+ feature_category :vulnerability_management
+ def perform; end
+ end
+ end
+end
+
+Gitlab::BackgroundMigration::CreateVulnerabilityLinks.prepend_mod
+# rubocop:enable Style/Documentation
diff --git a/lib/gitlab/background_migration/delete_orphaned_deployments.rb b/lib/gitlab/background_migration/delete_orphaned_deployments.rb
deleted file mode 100644
index 4a3a12ab53d..00000000000
--- a/lib/gitlab/background_migration/delete_orphaned_deployments.rb
+++ /dev/null
@@ -1,32 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module BackgroundMigration
- # Background migration for deleting orphaned deployments.
- class DeleteOrphanedDeployments
- include Database::MigrationHelpers
-
- def perform(start_id, end_id)
- orphaned_deployments
- .where(id: start_id..end_id)
- .delete_all
-
- mark_job_as_succeeded(start_id, end_id)
- end
-
- def orphaned_deployments
- define_batchable_model('deployments', connection: ApplicationRecord.connection)
- .where('NOT EXISTS (SELECT 1 FROM environments WHERE deployments.environment_id = environments.id)')
- end
-
- private
-
- def mark_job_as_succeeded(*arguments)
- Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded(
- self.class.name.demodulize,
- arguments
- )
- end
- end
- end
-end
diff --git a/lib/gitlab/background_migration/delete_orphaned_packages_dependencies.rb b/lib/gitlab/background_migration/delete_orphaned_packages_dependencies.rb
new file mode 100644
index 00000000000..a795300fa9d
--- /dev/null
+++ b/lib/gitlab/background_migration/delete_orphaned_packages_dependencies.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ # Deletes orphaned packages_dependencies records that have no packages_dependency_links
+ class DeleteOrphanedPackagesDependencies < BatchedMigrationJob
+ operation_name :delete_all
+ feature_category :package_registry
+
+ scope_to ->(relation) {
+ relation.where(
+ <<~SQL.squish
+ NOT EXISTS (
+ SELECT 1
+ FROM packages_dependency_links
+ WHERE packages_dependency_links.dependency_id = packages_dependencies.id
+ )
+ SQL
+ )
+ }
+
+ def perform
+ each_sub_batch(&:delete_all)
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/background_migration/disable_expiration_policies_linked_to_no_container_images.rb b/lib/gitlab/background_migration/disable_expiration_policies_linked_to_no_container_images.rb
deleted file mode 100644
index dad5da875ab..00000000000
--- a/lib/gitlab/background_migration/disable_expiration_policies_linked_to_no_container_images.rb
+++ /dev/null
@@ -1,41 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module BackgroundMigration
- BATCH_SIZE = 1000
-
- # This background migration disables container expiration policies connected
- # to a project that has no container repositories
- class DisableExpirationPoliciesLinkedToNoContainerImages
- # rubocop: disable Style/Documentation
- class ContainerExpirationPolicy < ActiveRecord::Base
- include EachBatch
-
- self.table_name = 'container_expiration_policies'
- end
- # rubocop: enable Style/Documentation
-
- def perform(from_id, to_id)
- ContainerExpirationPolicy.where(enabled: true, project_id: from_id..to_id).each_batch(of: BATCH_SIZE) do |batch|
- sql = <<-SQL
- WITH batched_relation AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (#{batch.select(:project_id).limit(BATCH_SIZE).to_sql})
- UPDATE container_expiration_policies
- SET enabled = FALSE
- FROM batched_relation
- WHERE container_expiration_policies.project_id = batched_relation.project_id
- AND NOT EXISTS (SELECT 1 FROM "container_repositories" WHERE container_repositories.project_id = container_expiration_policies.project_id)
- SQL
- execute(sql)
- end
- end
-
- private
-
- def execute(sql)
- ApplicationRecord
- .connection
- .execute(sql)
- end
- end
- end
-end
diff --git a/lib/gitlab/background_migration/disable_legacy_open_source_license_for_no_issues_no_repo_projects.rb b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_no_issues_no_repo_projects.rb
index 2eb7c5230ba..276c7a1c6fa 100644
--- a/lib/gitlab/background_migration/disable_legacy_open_source_license_for_no_issues_no_repo_projects.rb
+++ b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_no_issues_no_repo_projects.rb
@@ -23,8 +23,9 @@ module Gitlab
.joins('LEFT OUTER JOIN project_statistics ON project_statistics.project_id = projects.id')
.joins('LEFT OUTER JOIN project_settings ON project_settings.project_id = projects.id')
.joins('LEFT OUTER JOIN issues ON issues.project_id = projects.id')
- .where('project_statistics.repository_size' => 0,
- 'project_settings.legacy_open_source_license_available' => true)
+ .where(
+ 'project_statistics.repository_size' => 0,
+ 'project_settings.legacy_open_source_license_available' => true)
.group('projects.id')
.having('COUNT(issues.id) = 0')
diff --git a/lib/gitlab/background_migration/disable_legacy_open_source_license_for_one_member_no_repo_projects.rb b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_one_member_no_repo_projects.rb
index 8953836c705..7661ae4b5ad 100644
--- a/lib/gitlab/background_migration/disable_legacy_open_source_license_for_one_member_no_repo_projects.rb
+++ b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_one_member_no_repo_projects.rb
@@ -23,8 +23,9 @@ module Gitlab
.joins('LEFT OUTER JOIN project_statistics ON project_statistics.project_id = projects.id')
.joins('LEFT OUTER JOIN project_settings ON project_settings.project_id = projects.id')
.joins('LEFT OUTER JOIN project_authorizations ON project_authorizations.project_id = projects.id')
- .where('project_statistics.repository_size' => 0,
- 'project_settings.legacy_open_source_license_available' => true)
+ .where(
+ 'project_statistics.repository_size' => 0,
+ 'project_settings.legacy_open_source_license_available' => true)
.group('projects.id')
.having('COUNT(project_authorizations.user_id) = 1')
diff --git a/lib/gitlab/background_migration/drop_invalid_remediations.rb b/lib/gitlab/background_migration/drop_invalid_remediations.rb
deleted file mode 100644
index f0a0de586f5..00000000000
--- a/lib/gitlab/background_migration/drop_invalid_remediations.rb
+++ /dev/null
@@ -1,14 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module BackgroundMigration
- # rubocop: disable Style/Documentation
- class DropInvalidRemediations
- def perform(start_id, stop_id)
- end
- end
- # rubocop: enable Style/Documentation
- end
-end
-
-Gitlab::BackgroundMigration::DropInvalidRemediations.prepend_mod_with('Gitlab::BackgroundMigration::DropInvalidRemediations')
diff --git a/lib/gitlab/background_migration/drop_invalid_security_findings.rb b/lib/gitlab/background_migration/drop_invalid_security_findings.rb
deleted file mode 100644
index 000628e109c..00000000000
--- a/lib/gitlab/background_migration/drop_invalid_security_findings.rb
+++ /dev/null
@@ -1,47 +0,0 @@
-# frozen_string_literal: true
-module Gitlab
- module BackgroundMigration
- # Drop rows from security_findings where the uuid is NULL
- class DropInvalidSecurityFindings
- # rubocop:disable Style/Documentation
- class SecurityFinding < ActiveRecord::Base
- include ::EachBatch
- self.table_name = 'security_findings'
- scope :no_uuid, -> { where(uuid: nil) }
- end
- # rubocop:enable Style/Documentation
-
- PAUSE_SECONDS = 0.1
-
- def perform(start_id, end_id, sub_batch_size)
- ranged_query = SecurityFinding
- .where(id: start_id..end_id)
- .no_uuid
-
- ranged_query.each_batch(of: sub_batch_size) do |sub_batch|
- first, last = sub_batch.pick(Arel.sql('min(id), max(id)'))
-
- # The query need to be reconstructed because .each_batch modifies the default scope
- # See: https://gitlab.com/gitlab-org/gitlab/-/issues/330510
- SecurityFinding.unscoped
- .where(id: first..last)
- .no_uuid
- .delete_all
-
- sleep PAUSE_SECONDS
- end
-
- mark_job_as_succeeded(start_id, end_id, sub_batch_size)
- end
-
- private
-
- def mark_job_as_succeeded(*arguments)
- Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded(
- self.class.name.demodulize,
- arguments
- )
- end
- end
- end
-end
diff --git a/lib/gitlab/background_migration/drop_invalid_vulnerabilities.rb b/lib/gitlab/background_migration/drop_invalid_vulnerabilities.rb
deleted file mode 100644
index 293530f6536..00000000000
--- a/lib/gitlab/background_migration/drop_invalid_vulnerabilities.rb
+++ /dev/null
@@ -1,37 +0,0 @@
-# frozen_string_literal: true
-
-# rubocop: disable Style/Documentation
-class Gitlab::BackgroundMigration::DropInvalidVulnerabilities
- # rubocop: disable Gitlab/NamespacedClass
- class Vulnerability < ActiveRecord::Base
- self.table_name = "vulnerabilities"
- has_many :findings, class_name: 'VulnerabilitiesFinding', inverse_of: :vulnerability
- end
-
- class VulnerabilitiesFinding < ActiveRecord::Base
- self.table_name = "vulnerability_occurrences"
- belongs_to :vulnerability, class_name: 'Vulnerability', inverse_of: :findings, foreign_key: 'vulnerability_id'
- end
- # rubocop: enable Gitlab/NamespacedClass
-
- # rubocop: disable CodeReuse/ActiveRecord
- def perform(start_id, end_id)
- Vulnerability
- .where(id: start_id..end_id)
- .left_joins(:findings)
- .where(vulnerability_occurrences: { vulnerability_id: nil })
- .delete_all
-
- mark_job_as_succeeded(start_id, end_id)
- end
- # rubocop: enable CodeReuse/ActiveRecord
-
- private
-
- def mark_job_as_succeeded(*arguments)
- Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded(
- 'DropInvalidVulnerabilities',
- arguments
- )
- end
-end
diff --git a/lib/gitlab/background_migration/encrypt_ci_trigger_token.rb b/lib/gitlab/background_migration/encrypt_ci_trigger_token.rb
index b6e22e481fa..237c655a48a 100644
--- a/lib/gitlab/background_migration/encrypt_ci_trigger_token.rb
+++ b/lib/gitlab/background_migration/encrypt_ci_trigger_token.rb
@@ -18,8 +18,7 @@ module Gitlab
mode: :per_attribute_iv,
algorithm: 'aes-256-gcm',
key: Settings.attr_encrypted_db_key_base_32,
- encode: false,
- encode_vi: false
+ encode: false
before_save :copy_token_to_encrypted_token
diff --git a/lib/gitlab/background_migration/encrypt_integration_properties.rb b/lib/gitlab/background_migration/encrypt_integration_properties.rb
index c9582da2a51..28c28ae48eb 100644
--- a/lib/gitlab/background_migration/encrypt_integration_properties.rb
+++ b/lib/gitlab/background_migration/encrypt_integration_properties.rb
@@ -18,14 +18,14 @@ module Gitlab
scope :for_batch, ->(range) { where(id: range) }
attr_encrypted :encrypted_properties_tmp,
- attribute: :encrypted_properties,
- mode: :per_attribute_iv,
- key: ::Settings.attr_encrypted_db_key_base_32,
- algorithm: ALGORITHM,
- marshal: true,
- marshaler: ::Gitlab::Json,
- encode: false,
- encode_iv: false
+ attribute: :encrypted_properties,
+ mode: :per_attribute_iv,
+ key: ::Settings.attr_encrypted_db_key_base_32,
+ algorithm: ALGORITHM,
+ marshal: true,
+ marshaler: ::Gitlab::Json,
+ encode: false,
+ encode_iv: false
# See 'Integration#reencrypt_properties'
def encrypt_properties
diff --git a/lib/gitlab/background_migration/extract_project_topics_into_separate_table.rb b/lib/gitlab/background_migration/extract_project_topics_into_separate_table.rb
deleted file mode 100644
index 31b5b5cdb73..00000000000
--- a/lib/gitlab/background_migration/extract_project_topics_into_separate_table.rb
+++ /dev/null
@@ -1,63 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module BackgroundMigration
- # The class to extract the project topics into a separate `topics` table
- class ExtractProjectTopicsIntoSeparateTable
- # Temporary AR table for tags
- class Tag < ActiveRecord::Base
- self.table_name = 'tags'
- end
-
- # Temporary AR table for taggings
- class Tagging < ActiveRecord::Base
- self.table_name = 'taggings'
- belongs_to :tag
- end
-
- # Temporary AR table for topics
- class Topic < ActiveRecord::Base
- self.table_name = 'topics'
- end
-
- # Temporary AR table for project topics
- class ProjectTopic < ActiveRecord::Base
- self.table_name = 'project_topics'
- belongs_to :topic
- end
-
- # Temporary AR table for projects
- class Project < ActiveRecord::Base
- self.table_name = 'projects'
- end
-
- def perform(start_id, stop_id)
- Tagging.includes(:tag).where(taggable_type: 'Project', id: start_id..stop_id).each do |tagging|
- if Project.exists?(id: tagging.taggable_id) && tagging.tag
- begin
- topic = Topic.find_or_create_by(name: tagging.tag.name)
- project_topic = ProjectTopic.find_or_create_by(project_id: tagging.taggable_id, topic: topic)
-
- tagging.delete if project_topic.persisted?
- rescue StandardError => e
- Gitlab::ErrorTracking.log_exception(e, tagging_id: tagging.id)
- end
- else
- tagging.delete
- end
- end
-
- mark_job_as_succeeded(start_id, stop_id)
- end
-
- private
-
- def mark_job_as_succeeded(*arguments)
- Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded(
- self.class.name.demodulize,
- arguments
- )
- end
- end
- end
-end
diff --git a/lib/gitlab/background_migration/fix_incoherent_packages_size_on_project_statistics.rb b/lib/gitlab/background_migration/fix_incoherent_packages_size_on_project_statistics.rb
index 4b6bb12c91b..afd5e18ed7d 100644
--- a/lib/gitlab/background_migration/fix_incoherent_packages_size_on_project_statistics.rb
+++ b/lib/gitlab/background_migration/fix_incoherent_packages_size_on_project_statistics.rb
@@ -69,14 +69,14 @@ module Gitlab
self.table_name = 'packages_packages'
has_many :package_files,
- class_name: '::Gitlab::BackgroundMigration::FixIncoherentPackagesSizeOnProjectStatistics::PackageFile' # rubocop:disable Layout/LineLength
+ class_name: '::Gitlab::BackgroundMigration::FixIncoherentPackagesSizeOnProjectStatistics::PackageFile'
end
class PackageFile < ::ApplicationRecord
self.table_name = 'packages_package_files'
belongs_to :package,
- class_name: '::Gitlab::BackgroundMigration::FixIncoherentPackagesSizeOnProjectStatistics::Package' # rubocop:disable Layout/LineLength
+ class_name: '::Gitlab::BackgroundMigration::FixIncoherentPackagesSizeOnProjectStatistics::Package'
def self.sum_query
packages = FixIncoherentPackagesSizeOnProjectStatistics::Package.arel_table
diff --git a/lib/gitlab/background_migration/fix_merge_request_diff_commit_users.rb b/lib/gitlab/background_migration/fix_merge_request_diff_commit_users.rb
deleted file mode 100644
index 4df55a7b02a..00000000000
--- a/lib/gitlab/background_migration/fix_merge_request_diff_commit_users.rb
+++ /dev/null
@@ -1,21 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module BackgroundMigration
- # Background migration for fixing merge_request_diff_commit rows that don't
- # have committer/author details due to
- # https://gitlab.com/gitlab-org/gitlab/-/issues/344080.
- class FixMergeRequestDiffCommitUsers
- BATCH_SIZE = 100
-
- def initialize
- @commits = {}
- @users = {}
- end
-
- def perform(project_id)
- # No-op, see https://gitlab.com/gitlab-org/gitlab/-/issues/344540
- end
- end
- end
-end
diff --git a/lib/gitlab/background_migration/fix_vulnerability_reads_has_issues.rb b/lib/gitlab/background_migration/fix_vulnerability_reads_has_issues.rb
new file mode 100644
index 00000000000..5b3b5642ba8
--- /dev/null
+++ b/lib/gitlab/background_migration/fix_vulnerability_reads_has_issues.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ # This migration fixes existing `vulnerability_reads` records which did not have `has_issues`
+ # correctly set at the time of creation.
+ class FixVulnerabilityReadsHasIssues < BatchedMigrationJob
+ operation_name :fix_has_issues
+ feature_category :vulnerability_management
+
+ # rubocop:disable Style/Documentation
+ class VulnerabilityRead < ::ApplicationRecord
+ self.table_name = 'vulnerability_reads'
+
+ scope :with_vulnerability_ids, ->(ids) { where(vulnerability_id: ids) }
+ scope :without_issues, -> { where(has_issues: false) }
+ end
+ # rubocop:enable Style/Documentation
+
+ def perform
+ each_sub_batch do |sub_batch|
+ vulnerability_reads_with_issue_links(sub_batch).update_all('has_issues = true')
+ end
+ end
+
+ private
+
+ def vulnerability_reads_with_issue_links(sub_batch)
+ VulnerabilityRead.with_vulnerability_ids(sub_batch.select(:vulnerability_id)).without_issues
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/background_migration/issues_internal_id_scope_updater.rb b/lib/gitlab/background_migration/issues_internal_id_scope_updater.rb
new file mode 100644
index 00000000000..21ca4392003
--- /dev/null
+++ b/lib/gitlab/background_migration/issues_internal_id_scope_updater.rb
@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ # Migrates internal_ids records for `usage: issues` from project to namespace scope.
+ # For project issues it will be project namespace, for group issues it will be group namespace.
+ class IssuesInternalIdScopeUpdater < ::Gitlab::BackgroundMigration::BatchedMigrationJob
+ operation_name :issues_internal_id_scope_updater
+ feature_category :database
+
+ ISSUES_USAGE = 0 # see Enums::InternalId#usage_resources[:issues]
+
+ scope_to ->(relation) do
+ relation.where(usage: ISSUES_USAGE).where.not(project_id: nil)
+ end
+
+ def perform
+ each_sub_batch do |sub_batch|
+ create_namespace_scoped_records(sub_batch)
+ delete_project_scoped_records(sub_batch)
+ end
+ end
+
+ private
+
+ def delete_project_scoped_records(sub_batch)
+ # There is no need to keep the project scoped issues usage as we move to scoping issues to namespace.
+ # Also in case we do decide to move back to scoping issues usage to project, we are better off if the
+ # project record is not present as that would result in overlapping IIDs because project scoped issues
+ # usage will have outdated IIDs left in the DB
+ log_info("Deleted internal_ids records", ids: sub_batch.pluck(:id))
+
+ connection.execute(
+ <<~SQL
+ DELETE FROM internal_ids WHERE id IN (#{sub_batch.select(:id).to_sql})
+ SQL
+ )
+ end
+
+ def create_namespace_scoped_records(sub_batch)
+ # Creates a corresponding namespace scoped record for every `issues` usage scoped to a project.
+ # On conflict it means the record was already created when a new issue is created with the
+ # newly namespace scoped Issue model, see Issue#has_internal_id definition. In which case to
+ # make sure we have the namespace_id scoped record set to the greatest of the two last_values.
+ created_records_ids = connection.execute(
+ <<~SQL
+ INSERT INTO internal_ids (usage, last_value, namespace_id)
+ SELECT #{ISSUES_USAGE}, last_value, project_namespace_id
+ FROM internal_ids
+ INNER JOIN projects ON projects.id = internal_ids.project_id
+ WHERE internal_ids.id IN(#{sub_batch.select(:id).to_sql})
+ ON CONFLICT (usage, namespace_id) WHERE namespace_id IS NOT NULL
+ DO UPDATE SET last_value = GREATEST(EXCLUDED.last_value, internal_ids.last_value)
+ RETURNING id;
+ SQL
+ )
+
+ log_info("Created/updated internal_ids records", ids: created_records_ids.field_values('id'))
+ end
+
+ def log_info(message, **extra)
+ ::Gitlab::BackgroundMigration::Logger.info(migrator: self.class.to_s, message: message, **extra)
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/background_migration/logger.rb b/lib/gitlab/background_migration/logger.rb
index 4ea89771eff..d338c214140 100644
--- a/lib/gitlab/background_migration/logger.rb
+++ b/lib/gitlab/background_migration/logger.rb
@@ -4,6 +4,8 @@ module Gitlab
module BackgroundMigration
# Logger that can be used for migrations logging
class Logger < ::Gitlab::JsonLogger
+ exclude_context!
+
def self.file_name_noext
'migrations'
end
diff --git a/lib/gitlab/background_migration/migrate_evidences_for_vulnerability_findings.rb b/lib/gitlab/background_migration/migrate_evidences_for_vulnerability_findings.rb
new file mode 100644
index 00000000000..dd9fcf7fcfe
--- /dev/null
+++ b/lib/gitlab/background_migration/migrate_evidences_for_vulnerability_findings.rb
@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ # The class to migrate the evidence data into their own records from the json attribute
+ class MigrateEvidencesForVulnerabilityFindings < BatchedMigrationJob
+ feature_category :vulnerability_management
+ operation_name :migrate_evidences_for_vulnerability_findings
+
+ # The class is mimicking Vulnerabilites::Finding
+ class Finding < ApplicationRecord
+ self.table_name = 'vulnerability_occurrences'
+
+ validates :details, json_schema: { filename: 'vulnerability_finding_details', draft: 7 }, if: false
+ end
+
+ # The class is mimicking Vulnerabilites::Finding::Evidence
+ class Evidence < ApplicationRecord
+ self.table_name = 'vulnerability_finding_evidences'
+
+ # This data has been already validated when parsed into vulnerability_occurrences.raw_metadata
+ # Having this validation is a requerment from:
+ # https://gitlab.com/gitlab-org/gitlab/-/blob/dc3262f850cbd0ac14171d3c389b1258b4749cda/spec/db/schema_spec.rb#L253-265
+ validates :data, json_schema: { filename: "filename" }, if: false
+ end
+
+ def perform
+ each_sub_batch do |sub_batch|
+ migrate_evidences(sub_batch)
+ end
+ end
+
+ private
+
+ def migrate_evidences(sub_batch)
+ attrs = sub_batch.filter_map do |finding|
+ evidence = extract_evidence(finding.raw_metadata)
+
+ next unless evidence
+
+ build_evidence(finding, evidence)
+ end.compact
+
+ create_evidences(attrs) if attrs.present?
+ end
+
+ def build_evidence(finding, evidence)
+ current_time = Time.current
+ {
+ vulnerability_occurrence_id: finding.id,
+ data: evidence,
+ created_at: current_time,
+ updated_at: current_time
+ }
+ end
+
+ def create_evidences(evidences)
+ Evidence.upsert_all(evidences, returning: false, unique_by: %i[vulnerability_occurrence_id])
+ end
+
+ def extract_evidence(metadata)
+ # This is required because postgres doesn't support the null unicode character, i.e., \u0000.
+ # The following is the actual error:
+ # PG::UntranslatableCharacter: ERROR: unsupported Unicode escape sequence
+ # DETAIL: \u0000 cannot be converted to text.
+ return if metadata.include?('\u0000')
+
+ parsed_metadata = Gitlab::Json.parse(metadata)
+
+ parsed_metadata['evidence']
+ rescue JSON::ParserError
+ nil
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/background_migration/migrate_human_user_type.rb b/lib/gitlab/background_migration/migrate_human_user_type.rb
new file mode 100644
index 00000000000..2cb27225274
--- /dev/null
+++ b/lib/gitlab/background_migration/migrate_human_user_type.rb
@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ # Migrates all users with user_type = nil to user_type = 0
+ class MigrateHumanUserType < BatchedMigrationJob
+ OLD_TYPE_VALUE = nil
+ NEW_TYPE_VALUE = 0
+
+ operation_name :migrate_human_user_type
+ scope_to ->(relation) { relation.where(user_type: OLD_TYPE_VALUE) }
+ feature_category :user_management
+
+ def perform
+ cleanup_gin_indexes('users')
+
+ each_sub_batch do |sub_batch|
+ sub_batch.update_all(user_type: NEW_TYPE_VALUE)
+ end
+ end
+
+ private
+
+ def cleanup_gin_indexes(table_name)
+ sql = <<-SQL
+ SELECT indexname::text FROM pg_indexes WHERE tablename = '#{table_name}' AND indexdef ILIKE '%using gin%'
+ SQL
+
+ index_names = ApplicationRecord.connection.select_values(sql)
+
+ index_names.each do |index_name|
+ ApplicationRecord.connection.execute("SELECT gin_clean_pending_list('#{index_name}')")
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/background_migration/migrate_links_for_vulnerability_findings.rb b/lib/gitlab/background_migration/migrate_links_for_vulnerability_findings.rb
new file mode 100644
index 00000000000..0b79bc143db
--- /dev/null
+++ b/lib/gitlab/background_migration/migrate_links_for_vulnerability_findings.rb
@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ # The class to migrate the link data into their own records from the json attribute
+ class MigrateLinksForVulnerabilityFindings < BatchedMigrationJob
+ feature_category :vulnerability_management
+ operation_name :migrate_links_for_vulnerability_findings
+
+ # The class is mimicking Vulnerabilites::Finding
+ class Finding < ApplicationRecord
+ self.table_name = 'vulnerability_occurrences'
+
+ validates :details, json_schema: { filename: 'vulnerability_finding_details', draft: 7 }, if: false
+ end
+
+ # The class is mimicking Vulnerabilites::FindingLink
+ class Link < ApplicationRecord
+ self.table_name = 'vulnerability_finding_links'
+ end
+
+ def perform
+ each_sub_batch(batching_scope: ->(relation) { relation.select(:id, :raw_metadata) }) do |findings|
+ migrate_remediations(
+ findings,
+ Link
+ .where(vulnerability_occurrence_id: findings.map(&:id))
+ .group(:vulnerability_occurrence_id, :name, :url)
+ .count
+ )
+ end
+ end
+
+ private
+
+ def migrate_remediations(findings, existing_links)
+ findings.each do |finding|
+ create_links(build_links_from(finding, existing_links))
+ rescue ActiveRecord::StatementInvalid => e
+ logger.error(
+ message: e.message,
+ class: self.class.name,
+ model_id: finding.id
+ )
+ end
+ end
+
+ def build_link(finding, link)
+ current_time = Time.current
+ {
+ vulnerability_occurrence_id: finding.id,
+ name: link['name'],
+ url: link['url'],
+ created_at: current_time,
+ updated_at: current_time
+ }
+ end
+
+ def build_links_from(finding, existing_links)
+ extract_links(finding.raw_metadata).filter_map do |link|
+ key = [finding.id, link['name'], link['url']]
+ build_link(finding, link) unless existing_links.key?(key)
+ end
+ end
+
+ def create_links(attributes)
+ return if attributes.empty?
+
+ Link.upsert_all(attributes, returning: false)
+ end
+
+ def extract_links(metadata)
+ parsed_metadata = Gitlab::Json.parse(metadata)
+ parsed_links = Array.wrap(parsed_metadata['links'])
+
+ return [] if parsed_links.blank?
+
+ parsed_links.select { |link| link.try(:[], 'url').present? }.uniq
+ rescue JSON::ParserError => e
+ logger.warn(
+ message: e.message,
+ class: self.class.name
+ )
+ []
+ end
+
+ def logger
+ @logger ||= ::Gitlab::AppLogger
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users.rb b/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users.rb
deleted file mode 100644
index 7d150b9cd83..00000000000
--- a/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users.rb
+++ /dev/null
@@ -1,296 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module BackgroundMigration
- # Migrates author and committer names and emails from
- # merge_request_diff_commits to two columns that point to
- # merge_request_diff_commit_users.
- #
- # rubocop: disable Metrics/ClassLength
- class MigrateMergeRequestDiffCommitUsers
- # The number of user rows in merge_request_diff_commit_users to get in a
- # single query.
- USER_ROWS_PER_QUERY = 1_000
-
- # The number of rows in merge_request_diff_commits to get in a single
- # query.
- COMMIT_ROWS_PER_QUERY = 1_000
-
- # The number of rows in merge_request_diff_commits to update in a single
- # query.
- #
- # Tests in staging revealed that increasing the number of updates per
- # query translates to a longer total runtime for a migration. For example,
- # given the same range of rows to migrate, 1000 updates per query required
- # a total of roughly 15 seconds. On the other hand, 5000 updates per query
- # required a total of roughly 25 seconds. For this reason, we use a value
- # of 1000 rows per update.
- UPDATES_PER_QUERY = 1_000
-
- # rubocop: disable Style/Documentation
- class MergeRequestDiffCommit < ActiveRecord::Base
- include FromUnion
- extend ::SuppressCompositePrimaryKeyWarning
-
- self.table_name = 'merge_request_diff_commits'
-
- # Yields each row to migrate in the given range.
- #
- # This method uses keyset pagination to ensure we don't retrieve
- # potentially tens of thousands (or even hundreds of thousands) of rows
- # in a single query. Such queries could time out, or increase the amount
- # of memory needed to process the data.
- #
- # We can't use `EachBatch` and similar approaches, as
- # merge_request_diff_commits doesn't have a single monotonically
- # increasing primary key.
- def self.each_row_to_migrate(start_id, stop_id, &block)
- order = Pagination::Keyset::Order.build(
- %w[merge_request_diff_id relative_order].map do |col|
- Pagination::Keyset::ColumnOrderDefinition.new(
- attribute_name: col,
- order_expression: self.arel_table[col.to_sym].asc,
- nullable: :not_nullable,
- distinct: false
- )
- end
- )
-
- scope = MergeRequestDiffCommit
- .where(merge_request_diff_id: start_id...stop_id)
- .order(order)
-
- Pagination::Keyset::Iterator
- .new(scope: scope, use_union_optimization: true)
- .each_batch(of: COMMIT_ROWS_PER_QUERY) { |rows| rows.each(&block) }
- end
- end
- # rubocop: enable Style/Documentation
-
- # rubocop: disable Style/Documentation
- class MergeRequestDiffCommitUser < ActiveRecord::Base
- self.table_name = 'merge_request_diff_commit_users'
-
- def self.union(queries)
- from("(#{queries.join("\nUNION ALL\n")}) #{table_name}")
- end
- end
- # rubocop: enable Style/Documentation
-
- def perform(start_id, stop_id)
- return if already_processed?(start_id, stop_id)
-
- # This Hash maps user names + emails to their corresponding rows in
- # merge_request_diff_commit_users.
- user_mapping = {}
-
- user_details, diff_rows_to_update = get_data_to_update(start_id, stop_id)
-
- get_user_rows_in_batches(user_details, user_mapping)
- create_missing_users(user_details, user_mapping)
- update_commit_rows(diff_rows_to_update, user_mapping)
-
- Database::BackgroundMigrationJob.mark_all_as_succeeded(
- 'MigrateMergeRequestDiffCommitUsers',
- [start_id, stop_id]
- )
- end
-
- def already_processed?(start_id, stop_id)
- Database::BackgroundMigrationJob
- .for_migration_execution('MigrateMergeRequestDiffCommitUsers', [start_id, stop_id])
- .succeeded
- .any?
- end
-
- # Returns the data we'll use to determine what merge_request_diff_commits
- # rows to update, and what data to use for populating their
- # commit_author_id and committer_id columns.
- def get_data_to_update(start_id, stop_id)
- # This Set is used to retrieve users that already exist in
- # merge_request_diff_commit_users.
- users = Set.new
-
- # This Hash maps the primary key of every row in
- # merge_request_diff_commits to the (trimmed) author and committer
- # details to use for updating the row.
- to_update = {}
-
- MergeRequestDiffCommit.each_row_to_migrate(start_id, stop_id) do |row|
- author = [prepare(row.author_name), prepare(row.author_email)]
- committer = [prepare(row.committer_name), prepare(row.committer_email)]
-
- to_update[[row.merge_request_diff_id, row.relative_order]] =
- [author, committer]
-
- users << author if author[0] || author[1]
- users << committer if committer[0] || committer[1]
- end
-
- [users, to_update]
- end
-
- # Gets any existing rows in merge_request_diff_commit_users in batches.
- #
- # This method may end up having to retrieve lots of rows. To reduce the
- # overhead, we batch queries into a UNION query. We limit the number of
- # queries per UNION so we don't end up sending a single query containing
- # too many SELECT statements.
- def get_user_rows_in_batches(users, user_mapping)
- users.each_slice(USER_ROWS_PER_QUERY) do |pairs|
- queries = pairs.map do |(name, email)|
- MergeRequestDiffCommitUser.where(name: name, email: email).to_sql
- end
-
- MergeRequestDiffCommitUser.union(queries).each do |row|
- user_mapping[[row.name.to_s, row.email.to_s]] = row
- end
- end
- end
-
- # Creates any users for which no row exists in
- # merge_request_diff_commit_users.
- #
- # Not all users queried may exist yet, so we need to create any missing
- # ones; making sure we handle concurrent creations of the same user
- def create_missing_users(users, mapping)
- create = []
-
- users.each do |(name, email)|
- create << { name: name, email: email } unless mapping[[name, email]]
- end
-
- return if create.empty?
-
- MergeRequestDiffCommitUser
- .insert_all(create, returning: %w[id name email])
- .each do |row|
- mapping[[row['name'], row['email']]] = MergeRequestDiffCommitUser
- .new(id: row['id'], name: row['name'], email: row['email'])
- end
-
- # It's possible for (name, email) pairs to be inserted concurrently,
- # resulting in the above insert not returning anything. Here we get any
- # remaining users that were created concurrently.
- get_user_rows_in_batches(
- users.reject { |pair| mapping.key?(pair) },
- mapping
- )
- end
-
- # Updates rows in merge_request_diff_commits with their new
- # commit_author_id and committer_id values.
- def update_commit_rows(to_update, user_mapping)
- to_update.each_slice(UPDATES_PER_QUERY) do |slice|
- updates = {}
-
- slice.each do |(diff_id, order), (author, committer)|
- author_id = user_mapping[author]&.id
- committer_id = user_mapping[committer]&.id
-
- updates[[diff_id, order]] = [author_id, committer_id]
- end
-
- bulk_update_commit_rows(updates)
- end
- end
-
- # Bulk updates rows in the merge_request_diff_commits table with their new
- # author and/or committer ID values.
- #
- # Updates are batched together to reduce the overhead of having to produce
- # a single UPDATE for every row, as we may end up having to update
- # thousands of rows at once.
- #
- # The query produced by this method is along the lines of the following:
- #
- # UPDATE merge_request_diff_commits
- # SET commit_author_id =
- # CASE
- # WHEN (merge_request_diff_id, relative_order) = (x, y) THEN X
- # WHEN ...
- # END,
- # committer_id =
- # CASE
- # WHEN (merge_request_diff_id, relative_order) = (x, y) THEN Y
- # WHEN ...
- # END
- # WHERE (merge_request_diff_id, relative_order) IN ( (x, y), ... )
- #
- # The `mapping` argument is a Hash in the following format:
- #
- # { [merge_request_diff_id, relative_order] => [author_id, committer_id] }
- #
- # rubocop: disable Metrics/AbcSize
- def bulk_update_commit_rows(mapping)
- author_case = Arel::Nodes::Case.new
- committer_case = Arel::Nodes::Case.new
- primary_values = []
-
- mapping.each do |diff_id_and_order, (author_id, committer_id)|
- primary_value = Arel::Nodes::Grouping.new(diff_id_and_order)
-
- primary_values << primary_value
-
- if author_id
- author_case.when(primary_key.eq(primary_value)).then(author_id)
- end
-
- if committer_id
- committer_case.when(primary_key.eq(primary_value)).then(committer_id)
- end
- end
-
- if author_case.conditions.empty? && committer_case.conditions.empty?
- return
- end
-
- fields = []
-
- # Statements such as `SET x = CASE END` are not valid SQL statements, so
- # we omit setting an ID field if there are no values to populate it
- # with.
- if author_case.conditions.any?
- fields << [arel_table[:commit_author_id], author_case]
- end
-
- if committer_case.conditions.any?
- fields << [arel_table[:committer_id], committer_case]
- end
-
- query = Arel::UpdateManager.new
- .table(arel_table)
- .where(primary_key.in(primary_values))
- .set(fields)
- .to_sql
-
- MergeRequestDiffCommit.connection.execute(query)
- end
- # rubocop: enable Metrics/AbcSize
-
- def primary_key
- Arel::Nodes::Grouping.new(
- [arel_table[:merge_request_diff_id], arel_table[:relative_order]]
- )
- end
-
- def arel_table
- MergeRequestDiffCommit.arel_table
- end
-
- # Prepares a value to be inserted into a column in the table
- # `merge_request_diff_commit_users`. Values in this table are limited to
- # 512 characters.
- #
- # We treat empty strings as NULL values, as there's no point in (for
- # example) storing a row where both the name and Email are an empty
- # string. In addition, if we treated them differently we could end up with
- # two rows: one where field X is NULL, and one where field X is an empty
- # string. This is redundant, so we avoid storing such data.
- def prepare(value)
- value.present? ? value[0..511] : nil
- end
- end
- # rubocop: enable Metrics/ClassLength
- end
-end
diff --git a/lib/gitlab/background_migration/migrate_project_taggings_context_from_tags_to_topics.rb b/lib/gitlab/background_migration/migrate_project_taggings_context_from_tags_to_topics.rb
deleted file mode 100644
index 68bbd3cfebb..00000000000
--- a/lib/gitlab/background_migration/migrate_project_taggings_context_from_tags_to_topics.rb
+++ /dev/null
@@ -1,21 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module BackgroundMigration
- # The class to migrate the context of project taggings from `tags` to `topics`
- class MigrateProjectTaggingsContextFromTagsToTopics
- # Temporary AR table for taggings
- class Tagging < ActiveRecord::Base
- include EachBatch
-
- self.table_name = 'taggings'
- end
-
- def perform(start_id, stop_id)
- Tagging.where(taggable_type: 'Project', context: 'tags', id: start_id..stop_id).each_batch(of: 500) do |relation|
- relation.update_all(context: 'topics')
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/background_migration/migrate_remediations_for_vulnerability_findings.rb b/lib/gitlab/background_migration/migrate_remediations_for_vulnerability_findings.rb
new file mode 100644
index 00000000000..9eadef96db6
--- /dev/null
+++ b/lib/gitlab/background_migration/migrate_remediations_for_vulnerability_findings.rb
@@ -0,0 +1,164 @@
+# frozen_string_literal: true
+
+module Vulnerabilities
+ # The class is mimicking Vulnerabilites::Remediation
+ class Remediation < ApplicationRecord
+ include FileStoreMounter
+ include ShaAttribute
+
+ self.table_name = 'vulnerability_remediations'
+
+ sha_attribute :checksum
+
+ mount_file_store_uploader AttachmentUploader
+
+ def retrieve_upload(_identifier, paths)
+ Upload.find_by(model: self, path: paths)
+ end
+ end
+end
+
+module Gitlab
+ module BackgroundMigration
+ # The class to migrate the remediation data into their own records from the json attribute
+ class MigrateRemediationsForVulnerabilityFindings < BatchedMigrationJob
+ feature_category :vulnerability_management
+ operation_name :migrate_remediations_for_vulnerability_findings
+
+ # The class to encapsulate checksum and file for uploading
+ class DiffFile < StringIO
+ # This method is used by the `carrierwave` gem
+ def original_filename
+ @original_filename ||= self.class.original_filename(checksum)
+ end
+
+ def checksum
+ @checksum ||= self.class.checksum(string)
+ end
+
+ def self.checksum(value)
+ Digest::SHA256.hexdigest(value)
+ end
+
+ def self.original_filename(checksum)
+ "#{checksum}.diff"
+ end
+ end
+
+ # The class is mimicking Vulnerabilites::Finding
+ class Finding < ApplicationRecord
+ self.table_name = 'vulnerability_occurrences'
+
+ validates :details, json_schema: { filename: 'vulnerability_finding_details', draft: 7 }, if: false
+ end
+
+ # The class is mimicking Vulnerabilites::FindingRemediation
+ class FindingRemediation < ApplicationRecord
+ self.table_name = 'vulnerability_findings_remediations'
+ end
+
+ def perform
+ each_sub_batch do |sub_batch|
+ migrate_remediations(sub_batch)
+ end
+ end
+
+ private
+
+ def migrate_remediations(sub_batch)
+ sub_batch.each do |finding|
+ FindingRemediation.transaction do
+ remediations = append_remediations_diff_checksum(finding.raw_metadata)
+
+ result_ids = create_remediations(finding, remediations)
+
+ create_finding_remediations(finding.id, result_ids)
+ end
+ rescue StandardError => e
+ logger.error(
+ message: e.message,
+ class: self.class.name,
+ model_id: finding.id
+ )
+ end
+ end
+
+ def create_finding_remediations(finding_id, result_ids)
+ attrs = result_ids.map do |result_id|
+ build_finding_remediation_attrs(finding_id, result_id)
+ end
+
+ return unless attrs.present?
+
+ FindingRemediation.upsert_all(
+ attrs,
+ returning: false,
+ unique_by: [:vulnerability_occurrence_id, :vulnerability_remediation_id]
+ )
+ end
+
+ def create_remediations(finding, remediations)
+ attrs = remediations.map do |remediation|
+ build_remediation_attrs(finding, remediation)
+ end
+
+ return [] unless attrs.present?
+
+ ids_checksums = ::Vulnerabilities::Remediation.upsert_all(
+ attrs,
+ returning: %w[id checksum],
+ unique_by: [:project_id, :checksum]
+ )
+
+ ids_checksums.each do |id_checksum|
+ upload_file(id_checksum['id'], id_checksum['checksum'], remediations)
+ end
+
+ ids_checksums.pluck('id')
+ end
+
+ def upload_file(id, checksum, remediations)
+ deserialized_checksum = Gitlab::Database::ShaAttribute.new.deserialize(checksum)
+ diff = remediations.find { |rem| rem['checksum'] == deserialized_checksum }["diff"]
+ file = DiffFile.new(diff)
+ ::Vulnerabilities::Remediation.find_by(id: id).update!(file: file)
+ end
+
+ def build_remediation_attrs(finding, remediation)
+ {
+ project_id: finding.project_id,
+ summary: remediation['summary'],
+ file: DiffFile.original_filename(remediation['checksum']),
+ checksum: remediation['checksum'],
+ created_at: Time.current,
+ updated_at: Time.current
+ }
+ end
+
+ def build_finding_remediation_attrs(finding_id, remediation_id)
+ {
+ vulnerability_occurrence_id: finding_id,
+ vulnerability_remediation_id: remediation_id,
+ created_at: Time.current,
+ updated_at: Time.current
+ }
+ end
+
+ def append_remediations_diff_checksum(metadata)
+ parsed_metadata = Gitlab::Json.parse(metadata)
+
+ return [] unless parsed_metadata['remediations']
+
+ parsed_metadata['remediations'].filter_map do |remediation|
+ next unless remediation && remediation['diff'].present?
+
+ remediation.merge('checksum' => DiffFile.checksum(remediation['diff']))
+ end.compact.uniq
+ end
+
+ def logger
+ @logger ||= ::Gitlab::AppLogger
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/background_migration/migrate_shared_vulnerability_identifiers.rb b/lib/gitlab/background_migration/migrate_shared_vulnerability_identifiers.rb
new file mode 100644
index 00000000000..6a9f1692b72
--- /dev/null
+++ b/lib/gitlab/background_migration/migrate_shared_vulnerability_identifiers.rb
@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ # rubocop: disable Style/Documentation
+ class MigrateSharedVulnerabilityIdentifiers < BatchedMigrationJob
+ # rubocop: enable Style/Documentation
+
+ feature_category :vulnerability_management
+
+ def perform; end
+ end
+ end
+end
+
+# rubocop: disable Layout/LineLength
+Gitlab::BackgroundMigration::MigrateSharedVulnerabilityIdentifiers.prepend_mod_with("Gitlab::BackgroundMigration::MigrateSharedVulnerabilityIdentifiers")
+# rubocop: enable Layout/LineLength
diff --git a/lib/gitlab/background_migration/migrate_u2f_webauthn.rb b/lib/gitlab/background_migration/migrate_u2f_webauthn.rb
deleted file mode 100644
index 83aa36a11e6..00000000000
--- a/lib/gitlab/background_migration/migrate_u2f_webauthn.rb
+++ /dev/null
@@ -1,28 +0,0 @@
-# frozen_string_literal: true
-# rubocop:disable Style/Documentation
-
-module Gitlab
- module BackgroundMigration
- class MigrateU2fWebauthn
- class U2fRegistration < ActiveRecord::Base
- self.table_name = 'u2f_registrations'
- end
-
- class WebauthnRegistration < ActiveRecord::Base
- self.table_name = 'webauthn_registrations'
- end
-
- def perform(start_id, end_id)
- old_registrations = U2fRegistration.where(id: start_id..end_id)
- old_registrations.each_slice(100) do |slice|
- values = slice.map do |u2f_registration|
- converter = Gitlab::Auth::U2fWebauthnConverter.new(u2f_registration)
- converter.convert
- end
-
- WebauthnRegistration.insert_all(values, unique_by: :credential_xid, returning: false)
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature.rb b/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature.rb
deleted file mode 100644
index 06422ed282f..00000000000
--- a/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature.rb
+++ /dev/null
@@ -1,52 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module BackgroundMigration
- # This migration moves projects.container_registry_enabled values to
- # project_features.container_registry_access_level for the projects within
- # the given range of ids.
- class MoveContainerRegistryEnabledToProjectFeature
- MAX_BATCH_SIZE = 300
-
- ENABLED = 20
- DISABLED = 0
-
- def perform(from_id, to_id)
- (from_id..to_id).each_slice(MAX_BATCH_SIZE) do |batch|
- process_batch(batch.first, batch.last)
- end
-
- Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded('MoveContainerRegistryEnabledToProjectFeature', [from_id, to_id])
- end
-
- private
-
- def process_batch(from_id, to_id)
- ApplicationRecord.connection.execute(update_sql(from_id, to_id))
-
- logger.info(message: "#{self.class}: Copied container_registry_enabled values for projects with IDs between #{from_id}..#{to_id}")
- end
-
- # For projects that have a project_feature:
- # Set project_features.container_registry_access_level to ENABLED (20) or DISABLED (0)
- # depending if container_registry_enabled is true or false.
- def update_sql(from_id, to_id)
- <<~SQL
- UPDATE project_features
- SET container_registry_access_level = (CASE p.container_registry_enabled
- WHEN true THEN #{ENABLED}
- WHEN false THEN #{DISABLED}
- ELSE #{DISABLED}
- END)
- FROM projects p
- WHERE project_id = p.id AND
- project_id BETWEEN #{from_id} AND #{to_id}
- SQL
- end
-
- def logger
- @logger ||= Gitlab::BackgroundMigration::Logger.build
- end
- end
- end
-end
diff --git a/lib/gitlab/background_migration/populate_topics_total_projects_count_cache.rb b/lib/gitlab/background_migration/populate_topics_total_projects_count_cache.rb
deleted file mode 100644
index 2495cb51364..00000000000
--- a/lib/gitlab/background_migration/populate_topics_total_projects_count_cache.rb
+++ /dev/null
@@ -1,29 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module BackgroundMigration
- SUB_BATCH_SIZE = 1_000
-
- # The class to populates the total projects counter cache of topics
- class PopulateTopicsTotalProjectsCountCache
- # Temporary AR model for topics
- class Topic < ActiveRecord::Base
- include EachBatch
-
- self.table_name = 'topics'
- end
-
- def perform(start_id, stop_id)
- Topic.where(id: start_id..stop_id).each_batch(of: SUB_BATCH_SIZE) do |batch|
- ApplicationRecord.connection.execute(<<~SQL)
- WITH batched_relation AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (#{batch.select(:id).limit(SUB_BATCH_SIZE).to_sql})
- UPDATE topics
- SET total_projects_count = (SELECT COUNT(*) FROM project_topics WHERE topic_id = batched_relation.id)
- FROM batched_relation
- WHERE topics.id = batched_relation.id
- SQL
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/background_migration/populate_uuids_for_security_findings.rb b/lib/gitlab/background_migration/populate_uuids_for_security_findings.rb
deleted file mode 100644
index 175966b940d..00000000000
--- a/lib/gitlab/background_migration/populate_uuids_for_security_findings.rb
+++ /dev/null
@@ -1,18 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module BackgroundMigration
- # rubocop:disable Style/Documentation
- class PopulateUuidsForSecurityFindings
- NOP_RELATION = Class.new { def each_batch(*); end }
-
- def self.security_findings
- NOP_RELATION.new
- end
-
- def perform(*_scan_ids); end
- end
- end
-end
-
-Gitlab::BackgroundMigration::PopulateUuidsForSecurityFindings.prepend_mod_with('Gitlab::BackgroundMigration::PopulateUuidsForSecurityFindings')
diff --git a/lib/gitlab/background_migration/populate_vulnerability_dismissal_fields.rb b/lib/gitlab/background_migration/populate_vulnerability_dismissal_fields.rb
new file mode 100644
index 00000000000..ee0f73cc3de
--- /dev/null
+++ b/lib/gitlab/background_migration/populate_vulnerability_dismissal_fields.rb
@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ # Populates missing dismissal information for vulnerabilities.
+ class PopulateVulnerabilityDismissalFields < BatchedMigrationJob
+ feature_category :vulnerability_management
+ scope_to ->(relation) { relation.where('state = 2 AND (dismissed_at IS NULL OR dismissed_by_id IS NULL)') }
+ operation_name :populate_vulnerability_dismissal_fields
+
+ # rubocop:disable Style/Documentation
+ class Vulnerability < ApplicationRecord
+ self.table_name = 'vulnerabilities'
+
+ has_one :finding, class_name: 'Finding'
+
+ def copy_dismissal_information
+ return unless finding&.dismissal_feedback
+
+ update_columns(
+ dismissed_at: finding.dismissal_feedback.created_at,
+ dismissed_by_id: finding.dismissal_feedback.author_id
+ )
+ end
+ end
+
+ class Finding < ApplicationRecord
+ self.table_name = 'vulnerability_occurrences'
+
+ validates :details, json_schema: { filename: "filename" }
+
+ def dismissal_feedback
+ Feedback.dismissal.where(finding_uuid: uuid).first
+ end
+ end
+
+ class Feedback < ApplicationRecord
+ DISMISSAL_TYPE = 0 # dismissal
+
+ self.table_name = 'vulnerability_feedback'
+
+ scope :dismissal, -> { where(feedback_type: DISMISSAL_TYPE) }
+ end
+ # rubocop:enable Style/Documentation
+
+ def perform
+ each_sub_batch do |sub_batch|
+ vulnerability_ids = sub_batch.pluck(:id)
+ Vulnerability.includes(:finding).where(id: vulnerability_ids).each do |vulnerability|
+ populate_for(vulnerability)
+ end
+
+ log_info(vulnerability_ids)
+ end
+ end
+
+ private
+
+ def populate_for(vulnerability)
+ log_warning(vulnerability) unless vulnerability.copy_dismissal_information
+ rescue StandardError => error
+ log_error(error, vulnerability)
+ end
+
+ def log_info(vulnerability_ids)
+ ::Gitlab::BackgroundMigration::Logger.info(
+ migrator: self.class.name,
+ message: 'Dismissal information has been copied',
+ count: vulnerability_ids.length
+ )
+ end
+
+ def log_warning(vulnerability)
+ ::Gitlab::BackgroundMigration::Logger.warn(
+ migrator: self.class.name,
+ message: 'Could not update vulnerability!',
+ vulnerability_id: vulnerability.id
+ )
+ end
+
+ def log_error(error, vulnerability)
+ ::Gitlab::BackgroundMigration::Logger.error(
+ migrator: self.class.name,
+ message: error.message,
+ vulnerability_id: vulnerability.id
+ )
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/background_migration/remove_duplicate_vulnerabilities_findings.rb b/lib/gitlab/background_migration/remove_duplicate_vulnerabilities_findings.rb
deleted file mode 100644
index 15799659b55..00000000000
--- a/lib/gitlab/background_migration/remove_duplicate_vulnerabilities_findings.rb
+++ /dev/null
@@ -1,64 +0,0 @@
-# frozen_string_literal: true
-
-# rubocop: disable Style/Documentation
-class Gitlab::BackgroundMigration::RemoveDuplicateVulnerabilitiesFindings
- DELETE_BATCH_SIZE = 50
-
- # rubocop:disable Gitlab/NamespacedClass
- class VulnerabilitiesFinding < ActiveRecord::Base
- self.table_name = "vulnerability_occurrences"
- end
- # rubocop:enable Gitlab/NamespacedClass
-
- # rubocop:disable Gitlab/NamespacedClass
- class Vulnerability < ActiveRecord::Base
- self.table_name = "vulnerabilities"
- end
- # rubocop:enable Gitlab/NamespacedClass
-
- def perform(start_id, end_id)
- batch = VulnerabilitiesFinding.where(id: start_id..end_id)
-
- cte = Gitlab::SQL::CTE.new(:batch, batch.select(:report_type, :location_fingerprint, :primary_identifier_id, :project_id))
-
- query = VulnerabilitiesFinding
- .select('batch.report_type', 'batch.location_fingerprint', 'batch.primary_identifier_id', 'batch.project_id', 'array_agg(id) as ids')
- .distinct
- .with(cte.to_arel)
- .from(cte.alias_to(Arel.sql('batch')))
- .joins(
- %(
- INNER JOIN
- vulnerability_occurrences ON
- vulnerability_occurrences.report_type = batch.report_type AND
- vulnerability_occurrences.location_fingerprint = batch.location_fingerprint AND
- vulnerability_occurrences.primary_identifier_id = batch.primary_identifier_id AND
- vulnerability_occurrences.project_id = batch.project_id
- )).group('batch.report_type', 'batch.location_fingerprint', 'batch.primary_identifier_id', 'batch.project_id')
- .having('COUNT(*) > 1')
-
- ids_to_delete = []
-
- query.to_a.each do |record|
- # We want to keep the latest finding since it might have recent metadata
- duplicate_ids = record.ids.uniq.sort
- duplicate_ids.pop
- ids_to_delete.concat(duplicate_ids)
-
- if ids_to_delete.size == DELETE_BATCH_SIZE
- delete_findings_and_vulnerabilities(ids_to_delete)
- ids_to_delete.clear
- end
- end
-
- delete_findings_and_vulnerabilities(ids_to_delete) if ids_to_delete.any?
- end
-
- private
-
- def delete_findings_and_vulnerabilities(ids)
- vulnerability_ids = VulnerabilitiesFinding.where(id: ids).pluck(:vulnerability_id).compact
- VulnerabilitiesFinding.where(id: ids).delete_all
- Vulnerability.where(id: vulnerability_ids).delete_all
- end
-end
diff --git a/lib/gitlab/background_migration/remove_occurrence_pipelines_and_duplicate_vulnerabilities_findings.rb b/lib/gitlab/background_migration/remove_occurrence_pipelines_and_duplicate_vulnerabilities_findings.rb
index 7fe5a427d10..f4f54e2b2eb 100644
--- a/lib/gitlab/background_migration/remove_occurrence_pipelines_and_duplicate_vulnerabilities_findings.rb
+++ b/lib/gitlab/background_migration/remove_occurrence_pipelines_and_duplicate_vulnerabilities_findings.rb
@@ -53,7 +53,7 @@ class Gitlab::BackgroundMigration::RemoveOccurrencePipelinesAndDuplicateVulnerab
def mark_job_as_succeeded(*arguments)
Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded(
self.class.name.demodulize,
- arguments
+ arguments
)
end
end
diff --git a/lib/gitlab/background_migration/remove_project_group_link_with_missing_groups.rb b/lib/gitlab/background_migration/remove_project_group_link_with_missing_groups.rb
new file mode 100644
index 00000000000..879e52c96bf
--- /dev/null
+++ b/lib/gitlab/background_migration/remove_project_group_link_with_missing_groups.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ # A job to remove `project_group_links` records whose associated group
+ # does not exist in `namespaces` table anymore.
+ class RemoveProjectGroupLinkWithMissingGroups < Gitlab::BackgroundMigration::BatchedMigrationJob
+ scope_to ->(relation) { relation }
+ operation_name :delete_all
+ feature_category :subgroups
+
+ def perform
+ each_sub_batch do |sub_batch|
+ records = sub_batch.joins(
+ "LEFT OUTER JOIN namespaces ON namespaces.id = project_group_links.group_id AND namespaces.type = 'Group'"
+ ).where(namespaces: { id: nil })
+
+ ids = records.map(&:id)
+
+ next if ids.empty?
+
+ Gitlab::AppLogger.info({ message: 'Removing project group link with non-existent groups',
+ deleted_count: ids.count,
+ ids: ids })
+
+ records.delete_all
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/background_migration/reset_status_on_container_repositories.rb b/lib/gitlab/background_migration/reset_status_on_container_repositories.rb
index 0dbe2781327..56506814dc0 100644
--- a/lib/gitlab/background_migration/reset_status_on_container_repositories.rb
+++ b/lib/gitlab/background_migration/reset_status_on_container_repositories.rb
@@ -36,8 +36,8 @@ module Gitlab
included do
has_one :route,
- as: :source,
- class_name: '::Gitlab::BackgroundMigration::ResetStatusOnContainerRepositories::Route'
+ as: :source,
+ class_name: '::Gitlab::BackgroundMigration::ResetStatusOnContainerRepositories::Route'
end
def full_path
@@ -67,7 +67,7 @@ module Gitlab
self.inheritance_column = :_type_disabled
belongs_to :parent,
- class_name: '::Gitlab::BackgroundMigration::ResetStatusOnContainerRepositories::Namespace'
+ class_name: '::Gitlab::BackgroundMigration::ResetStatusOnContainerRepositories::Namespace'
def self.polymorphic_name
'Namespace'
@@ -80,7 +80,7 @@ module Gitlab
self.table_name = 'projects'
belongs_to :namespace,
- class_name: '::Gitlab::BackgroundMigration::ResetStatusOnContainerRepositories::Namespace'
+ class_name: '::Gitlab::BackgroundMigration::ResetStatusOnContainerRepositories::Namespace'
alias_method :parent, :namespace
alias_attribute :parent_id, :namespace_id
@@ -92,7 +92,7 @@ module Gitlab
self.table_name = 'container_repositories'
belongs_to :project,
- class_name: '::Gitlab::BackgroundMigration::ResetStatusOnContainerRepositories::Project'
+ class_name: '::Gitlab::BackgroundMigration::ResetStatusOnContainerRepositories::Project'
def tags?
result = ContainerRegistry.tags_for(path).any?
diff --git a/lib/gitlab/background_migration/steal_migrate_merge_request_diff_commit_users.rb b/lib/gitlab/background_migration/steal_migrate_merge_request_diff_commit_users.rb
deleted file mode 100644
index 43a7032e682..00000000000
--- a/lib/gitlab/background_migration/steal_migrate_merge_request_diff_commit_users.rb
+++ /dev/null
@@ -1,33 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module BackgroundMigration
- # A background migration that finished any pending
- # MigrateMergeRequestDiffCommitUsers jobs, and schedules new jobs itself.
- #
- # This migration exists so we can bypass rescheduling issues (e.g. jobs
- # getting dropped after too many retries) that may occur when
- # MigrateMergeRequestDiffCommitUsers jobs take longer than expected.
- class StealMigrateMergeRequestDiffCommitUsers
- def perform(start_id, stop_id)
- MigrateMergeRequestDiffCommitUsers.new.perform(start_id, stop_id)
- schedule_next_job
- end
-
- def schedule_next_job
- next_job = Database::BackgroundMigrationJob
- .for_migration_class('MigrateMergeRequestDiffCommitUsers')
- .pending
- .first
-
- return unless next_job
-
- BackgroundMigrationWorker.perform_in(
- 5.minutes,
- 'StealMigrateMergeRequestDiffCommitUsers',
- next_job.arguments
- )
- end
- end
- end
-end
diff --git a/lib/gitlab/background_migration/update_timelogs_project_id.rb b/lib/gitlab/background_migration/update_timelogs_project_id.rb
deleted file mode 100644
index 69bb5cf6e6d..00000000000
--- a/lib/gitlab/background_migration/update_timelogs_project_id.rb
+++ /dev/null
@@ -1,44 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module BackgroundMigration
- # Class to populate project_id for timelogs
- class UpdateTimelogsProjectId
- BATCH_SIZE = 1000
-
- def perform(start_id, stop_id)
- (start_id..stop_id).step(BATCH_SIZE).each do |offset|
- update_issue_timelogs(offset, offset + BATCH_SIZE)
- update_merge_request_timelogs(offset, offset + BATCH_SIZE)
- end
- end
-
- def update_issue_timelogs(batch_start, batch_stop)
- execute(<<~SQL)
- UPDATE timelogs
- SET project_id = issues.project_id
- FROM issues
- WHERE issues.id = timelogs.issue_id
- AND timelogs.id BETWEEN #{batch_start} AND #{batch_stop}
- AND timelogs.project_id IS NULL;
- SQL
- end
-
- def update_merge_request_timelogs(batch_start, batch_stop)
- execute(<<~SQL)
- UPDATE timelogs
- SET project_id = merge_requests.target_project_id
- FROM merge_requests
- WHERE merge_requests.id = timelogs.merge_request_id
- AND timelogs.id BETWEEN #{batch_start} AND #{batch_stop}
- AND timelogs.project_id IS NULL;
- SQL
- end
-
- def execute(sql)
- @connection ||= ApplicationRecord.connection
- @connection.execute(sql)
- end
- end
- end
-end
diff --git a/lib/gitlab/background_migration/update_users_where_two_factor_auth_required_from_group.rb b/lib/gitlab/background_migration/update_users_where_two_factor_auth_required_from_group.rb
deleted file mode 100644
index 10db9f5064a..00000000000
--- a/lib/gitlab/background_migration/update_users_where_two_factor_auth_required_from_group.rb
+++ /dev/null
@@ -1,129 +0,0 @@
-# frozen_string_literal: true
-# rubocop:disable Style/Documentation
-
-module Gitlab
- module BackgroundMigration
- class UpdateUsersWhereTwoFactorAuthRequiredFromGroup # rubocop:disable Metrics/ClassLength
- def perform(start_id, stop_id)
- ApplicationRecord.connection.execute <<~SQL
- UPDATE
- users
- SET
- require_two_factor_authentication_from_group = TRUE
- WHERE
- users.id BETWEEN #{start_id}
- AND #{stop_id}
- AND users.require_two_factor_authentication_from_group = FALSE
- AND users.id IN (
- SELECT
- DISTINCT users_groups_query.user_id
- FROM
- (
- SELECT
- users.id AS user_id,
- members.source_id AS group_ids
- FROM
- users
- LEFT JOIN members ON members.source_type = 'Namespace'
- AND members.requested_at IS NULL
- AND members.user_id = users.id
- AND members.type = 'GroupMember'
- WHERE
- users.require_two_factor_authentication_from_group = FALSE
- AND users.id BETWEEN #{start_id}
- AND #{stop_id}) AS users_groups_query
- INNER JOIN LATERAL (
- WITH RECURSIVE "base_and_ancestors" AS (
- (
- SELECT
- "namespaces"."type",
- "namespaces"."id",
- "namespaces"."parent_id",
- "namespaces"."require_two_factor_authentication"
- FROM
- "namespaces"
- WHERE
- "namespaces"."type" = 'Group'
- AND "namespaces"."id" = users_groups_query.group_ids
- )
- UNION
- (
- SELECT
- "namespaces"."type",
- "namespaces"."id",
- "namespaces"."parent_id",
- "namespaces"."require_two_factor_authentication"
- FROM
- "namespaces",
- "base_and_ancestors"
- WHERE
- "namespaces"."type" = 'Group'
- AND "namespaces"."id" = "base_and_ancestors"."parent_id"
- )
- ),
- "base_and_descendants" AS (
- (
- SELECT
- "namespaces"."type",
- "namespaces"."id",
- "namespaces"."parent_id",
- "namespaces"."require_two_factor_authentication"
- FROM
- "namespaces"
- WHERE
- "namespaces"."type" = 'Group'
- AND "namespaces"."id" = users_groups_query.group_ids
- )
- UNION
- (
- SELECT
- "namespaces"."type",
- "namespaces"."id",
- "namespaces"."parent_id",
- "namespaces"."require_two_factor_authentication"
- FROM
- "namespaces",
- "base_and_descendants"
- WHERE
- "namespaces"."type" = 'Group'
- AND "namespaces"."parent_id" = "base_and_descendants"."id"
- )
- )
- SELECT
- "namespaces".*
- FROM
- (
- (
- SELECT
- "namespaces"."type",
- "namespaces"."id",
- "namespaces"."parent_id",
- "namespaces"."require_two_factor_authentication"
- FROM
- "base_and_ancestors" AS "namespaces"
- WHERE
- "namespaces"."type" = 'Group'
- )
- UNION
- (
- SELECT
- "namespaces"."type",
- "namespaces"."id",
- "namespaces"."parent_id",
- "namespaces"."require_two_factor_authentication"
- FROM
- "base_and_descendants" AS "namespaces"
- WHERE
- "namespaces"."type" = 'Group'
- )
- ) namespaces
- WHERE
- "namespaces"."type" = 'Group'
- AND "namespaces"."require_two_factor_authentication" = TRUE
- ) AS hierarchy_tree ON TRUE
- );
- SQL
- end
- end
- end
-end
diff --git a/lib/gitlab/background_migration/update_vulnerability_occurrences_location.rb b/lib/gitlab/background_migration/update_vulnerability_occurrences_location.rb
deleted file mode 100644
index 458e0537f1c..00000000000
--- a/lib/gitlab/background_migration/update_vulnerability_occurrences_location.rb
+++ /dev/null
@@ -1,14 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module BackgroundMigration
- # rubocop: disable Style/Documentation
- class UpdateVulnerabilityOccurrencesLocation
- def perform(start_id, stop_id)
- end
- end
- # rubocop: enable Style/Documentation
- end
-end
-
-Gitlab::BackgroundMigration::UpdateVulnerabilityOccurrencesLocation.prepend_mod_with('Gitlab::BackgroundMigration::UpdateVulnerabilityOccurrencesLocation')
diff --git a/lib/gitlab/backup_logger.rb b/lib/gitlab/backup_logger.rb
index fad36b860ae..ec85c55d4a4 100644
--- a/lib/gitlab/backup_logger.rb
+++ b/lib/gitlab/backup_logger.rb
@@ -2,6 +2,8 @@
module Gitlab
class BackupLogger < Gitlab::JsonLogger
+ exclude_context!
+
def self.file_name_noext
'backup_json'
end
diff --git a/lib/gitlab/bare_repository_import/importer.rb b/lib/gitlab/bare_repository_import/importer.rb
deleted file mode 100644
index 28f1a10f9a7..00000000000
--- a/lib/gitlab/bare_repository_import/importer.rb
+++ /dev/null
@@ -1,132 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module BareRepositoryImport
- class Importer
- NoAdminError = Class.new(StandardError)
-
- def self.execute(import_path)
- unless import_path.ends_with?('/')
- import_path = "#{import_path}/"
- end
-
- repos_to_import = Dir.glob(import_path + '**/*.git')
-
- unless user = User.admins.order_id_asc.first
- raise NoAdminError, 'No admin user found to import repositories'
- end
-
- repos_to_import.each do |repo_path|
- bare_repo = Gitlab::BareRepositoryImport::Repository.new(import_path, repo_path)
-
- unless bare_repo.processable?
- log " * Skipping repo #{bare_repo.repo_path}".color(:yellow)
-
- next
- end
-
- log "Processing #{repo_path}".color(:yellow)
-
- new(user, bare_repo).create_project_if_needed
- end
- end
-
- # This is called from within a rake task only used by Admins, so allow writing
- # to STDOUT
- def self.log(message)
- puts message # rubocop:disable Rails/Output
- end
-
- attr_reader :user, :project_name, :bare_repo
-
- delegate :log, to: :class
- delegate :project_name, :project_full_path, :group_path, :repo_path, :wiki_path, to: :bare_repo
-
- def initialize(user, bare_repo)
- @user = user
- @bare_repo = bare_repo
- end
-
- def create_project_if_needed
- if project = Project.find_by_full_path(project_full_path)
- log " * #{project.name} (#{project_full_path}) exists"
-
- return project
- end
-
- create_project
- end
-
- private
-
- def create_project
- group = find_or_create_groups
-
- project = Projects::CreateService.new(user,
- name: project_name,
- path: project_name,
- skip_disk_validation: true,
- skip_wiki: bare_repo.wiki_exists?,
- import_type: 'bare_repository',
- namespace_id: group&.id).execute
-
- if project.persisted? && mv_repositories(project)
- log " * Created #{project.name} (#{project_full_path})".color(:green)
-
- project.set_full_path
-
- ProjectCacheWorker.perform_async(project.id)
- else
- log " * Failed trying to create #{project.name} (#{project_full_path})".color(:red)
- log " Errors: #{project.errors.messages}".color(:red) if project.errors.any?
- end
-
- project
- end
-
- def mv_repositories(project)
- mv_repo(bare_repo.repo_path, project.repository)
-
- if bare_repo.wiki_exists?
- mv_repo(bare_repo.wiki_path, project.wiki.repository)
- end
-
- true
- rescue StandardError => e
- log " * Failed to move repo: #{e.message}".color(:red)
-
- false
- end
-
- def mv_repo(path, repository)
- repository.create_from_bundle(bundle(path))
- FileUtils.rm_rf(path)
- end
-
- def storage_path_for_shard(shard)
- Gitlab.config.repositories.storages[shard].legacy_disk_path
- end
-
- def find_or_create_groups
- return unless group_path.present?
-
- log " * Using namespace: #{group_path}"
-
- Groups::NestedCreateService.new(user, group_path: group_path).execute
- end
-
- def bundle(repo_path)
- # TODO: we could save some time and disk space by using
- # `git bundle create - --all` and streaming the bundle directly to
- # Gitaly, rather than writing it on disk first
- bundle_path = "#{repo_path}.bundle"
- cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{repo_path} bundle create #{bundle_path} --all)
- output, status = Gitlab::Popen.popen(cmd)
-
- raise output unless status == 0
-
- bundle_path
- end
- end
- end
-end
diff --git a/lib/gitlab/bare_repository_import/repository.rb b/lib/gitlab/bare_repository_import/repository.rb
deleted file mode 100644
index b903c581aac..00000000000
--- a/lib/gitlab/bare_repository_import/repository.rb
+++ /dev/null
@@ -1,72 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module BareRepositoryImport
- class Repository
- include ::Gitlab::Utils::StrongMemoize
-
- attr_reader :group_path, :project_name, :repo_path
-
- def initialize(root_path, repo_path)
- unless root_path.ends_with?('/')
- root_path = "#{root_path}/"
- end
-
- @root_path = root_path
- @repo_path = repo_path
-
- full_path =
- if hashed? && !wiki?
- repository.config.get('gitlab.fullpath')
- else
- repo_relative_path
- end
-
- # Split path into 'all/the/namespaces' and 'project_name'
- @group_path, _, @project_name = full_path.to_s.rpartition('/')
- end
-
- def wiki_exists?
- File.exist?(wiki_path)
- end
-
- def wiki_path
- @wiki_path ||= repo_path.sub(/\.git$/, '.wiki.git')
- end
-
- def project_full_path
- @project_full_path ||= "#{group_path}/#{project_name}"
- end
-
- def processable?
- return false if wiki?
- return false if hashed? && (group_path.blank? || project_name.blank?)
-
- true
- end
-
- private
-
- def wiki?
- strong_memoize(:wiki) do
- repo_path.end_with?('.wiki.git')
- end
- end
-
- def hashed?
- strong_memoize(:hashed) do
- repo_relative_path.include?('@hashed')
- end
- end
-
- def repo_relative_path
- # Remove root path and `.git` at the end
- repo_path[@root_path.size...-4]
- end
-
- def repository
- @repository ||= Rugged::Repository.new(repo_path)
- end
- end
- end
-end
diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb
index 3dafe7c8962..592e75b1430 100644
--- a/lib/gitlab/bitbucket_import/importer.rb
+++ b/lib/gitlab/bitbucket_import/importer.rb
@@ -72,7 +72,7 @@ module Gitlab
return unless last_bitbucket_issue
- Issue.track_project_iid!(project, last_bitbucket_issue.iid)
+ Issue.track_namespace_iid!(project.project_namespace, last_bitbucket_issue.iid)
end
def repo
diff --git a/lib/gitlab/bullet/exclusions.rb b/lib/gitlab/bullet/exclusions.rb
index f897ff492d9..406d0a80a07 100644
--- a/lib/gitlab/bullet/exclusions.rb
+++ b/lib/gitlab/bullet/exclusions.rb
@@ -27,7 +27,8 @@ module Gitlab
def exclusions
@exclusions ||= if File.exist?(config_file)
- YAML.load_file(config_file)['exclusions']&.values || []
+ config = YAML.safe_load_file(config_file, permitted_classes: [Range])
+ config['exclusions']&.values || []
else
[]
end
diff --git a/lib/gitlab/cache/client.rb b/lib/gitlab/cache/client.rb
new file mode 100644
index 00000000000..37d6cac8d43
--- /dev/null
+++ b/lib/gitlab/cache/client.rb
@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Cache
+ # It replaces Rails.cache with metrics support
+ class Client
+ DEFAULT_BACKING_RESOURCE = :unknown
+
+ # Build Cache client with the metadata support
+ #
+ # @param cache_identifier [String] defines the location of the cache definition
+ # Example: "ProtectedBranches::CacheService#fetch"
+ # @param feature_category [Symbol] name of the feature category (from config/feature_categories.yml)
+ # @param backing_resource [Symbol] most affected resource by cache generation (full list: VALID_BACKING_RESOURCES)
+ # @return [Gitlab::Cache::Client]
+ def self.build_with_metadata(
+ cache_identifier:,
+ feature_category:,
+ backing_resource: DEFAULT_BACKING_RESOURCE
+ )
+ new(Metadata.new(
+ cache_identifier: cache_identifier,
+ feature_category: feature_category,
+ backing_resource: backing_resource
+ ))
+ end
+
+ def initialize(metadata, backend: Rails.cache)
+ @metadata = metadata
+ @metrics = Metrics.new(metadata)
+ @backend = backend
+ end
+
+ def read(name)
+ read_result = backend.read(name)
+
+ if read_result.nil?
+ metrics.increment_cache_miss
+ else
+ metrics.increment_cache_hit
+ end
+
+ read_result
+ end
+
+ def fetch(name, options = nil, &block)
+ read_result = read(name)
+
+ return read_result unless block || read_result
+
+ backend.fetch(name, options) do
+ metrics.observe_cache_generation(&block)
+ end
+ end
+
+ delegate :write, :exist?, :delete, to: :backend
+
+ attr_reader :metadata, :metrics
+
+ private
+
+ attr_reader :backend
+ end
+ end
+end
diff --git a/lib/gitlab/cache/metadata.rb b/lib/gitlab/cache/metadata.rb
index d6c89b5b2c3..de35b332300 100644
--- a/lib/gitlab/cache/metadata.rb
+++ b/lib/gitlab/cache/metadata.rb
@@ -5,21 +5,23 @@ module Gitlab
# Value object for cache metadata
class Metadata
VALID_BACKING_RESOURCES = [:cpu, :database, :gitaly, :memory, :unknown].freeze
- DEFAULT_BACKING_RESOURCE = :unknown
+ # @param cache_identifier [String] defines the location of the cache definition
+ # Example: "ProtectedBranches::CacheService#fetch"
+ # @param feature_category [Symbol] name of the feature category (from config/feature_categories.yml)
+ # @param backing_resource [Symbol] most affected resource by cache generation (full list: VALID_BACKING_RESOURCES)
+ # @return [Gitlab::Cache::Metadata]
def initialize(
cache_identifier:,
feature_category:,
- caller_id: Gitlab::ApplicationContext.current_context_attribute(:caller_id),
- backing_resource: DEFAULT_BACKING_RESOURCE
+ backing_resource: Client::DEFAULT_BACKING_RESOURCE
)
@cache_identifier = cache_identifier
@feature_category = Gitlab::FeatureCategories.default.get!(feature_category)
- @caller_id = caller_id
@backing_resource = fetch_backing_resource!(backing_resource)
end
- attr_reader :caller_id, :cache_identifier, :feature_category, :backing_resource
+ attr_reader :cache_identifier, :feature_category, :backing_resource
private
@@ -28,7 +30,7 @@ module Gitlab
raise "Unknown backing resource: #{resource}" if Gitlab.dev_or_test_env?
- DEFAULT_BACKING_RESOURCE
+ Client::DEFAULT_BACKING_RESOURCE
end
end
end
diff --git a/lib/gitlab/cache/metrics.rb b/lib/gitlab/cache/metrics.rb
index 00d4e6e4d4e..d9c80f076b9 100644
--- a/lib/gitlab/cache/metrics.rb
+++ b/lib/gitlab/cache/metrics.rb
@@ -58,7 +58,6 @@ module Gitlab
def labels
@labels ||= {
- caller_id: cache_metadata.caller_id,
cache_identifier: cache_metadata.cache_identifier,
feature_category: cache_metadata.feature_category,
backing_resource: cache_metadata.backing_resource
diff --git a/lib/gitlab/changes_list.rb b/lib/gitlab/changes_list.rb
index 999d2ee4356..ca55692ca2f 100644
--- a/lib/gitlab/changes_list.rb
+++ b/lib/gitlab/changes_list.rb
@@ -15,12 +15,12 @@ module Gitlab
end
def changes
- @changes ||= @raw_changes.map do |change|
+ @changes ||= @raw_changes.filter_map do |change|
next if change.blank?
oldrev, newrev, ref = change.strip.split(' ')
{ oldrev: oldrev, newrev: newrev, ref: ref }
- end.compact
+ end
end
end
end
diff --git a/lib/gitlab/chat/responder.rb b/lib/gitlab/chat/responder.rb
index 478be5bd350..7ec64b7cfbf 100644
--- a/lib/gitlab/chat/responder.rb
+++ b/lib/gitlab/chat/responder.rb
@@ -11,21 +11,13 @@ module Gitlab
#
# build - A `Ci::Build` that executed a chat command.
def self.responder_for(build)
- if Feature.enabled?(:use_response_url_for_chat_responder)
- response_url = build.pipeline.chat_data&.response_url
- return unless response_url
+ response_url = build.pipeline.chat_data&.response_url
+ return unless response_url
- if response_url.start_with?('https://hooks.slack.com/')
- Gitlab::Chat::Responder::Slack.new(build)
- else
- Gitlab::Chat::Responder::Mattermost.new(build)
- end
+ if response_url.start_with?('https://hooks.slack.com/')
+ Gitlab::Chat::Responder::Slack.new(build)
else
- integration = build.pipeline.chat_data&.chat_name&.integration
-
- if (responder = integration.try(:chat_responder))
- responder.new(build)
- end
+ Gitlab::Chat::Responder::Mattermost.new(build)
end
end
end
diff --git a/lib/gitlab/checks/base_single_checker.rb b/lib/gitlab/checks/base_single_checker.rb
index 435f4ccf5ba..755778efa60 100644
--- a/lib/gitlab/checks/base_single_checker.rb
+++ b/lib/gitlab/checks/base_single_checker.rb
@@ -5,7 +5,7 @@ module Gitlab
class BaseSingleChecker < BaseChecker
attr_reader :change_access
- delegate(*SingleChangeAccess::ATTRIBUTES, to: :change_access)
+ delegate(*SingleChangeAccess::ATTRIBUTES, :branch_ref?, :tag_ref?, to: :change_access)
def initialize(change_access)
@change_access = change_access
diff --git a/lib/gitlab/checks/changes_access.rb b/lib/gitlab/checks/changes_access.rb
index 99752dc6a01..194e3f6e938 100644
--- a/lib/gitlab/checks/changes_access.rb
+++ b/lib/gitlab/checks/changes_access.rb
@@ -36,13 +36,13 @@ module Gitlab
# any of the new revisions.
def commits
strong_memoize(:commits) do
- newrevs = @changes.map do |change|
+ newrevs = @changes.filter_map do |change|
newrev = change[:newrev]
next if blank_rev?(newrev)
newrev
- end.compact
+ end
next [] if newrevs.empty?
@@ -89,7 +89,7 @@ module Gitlab
@single_changes_accesses ||=
changes.map do |change|
commits =
- if blank_rev?(change[:newrev])
+ if !commitish_ref?(change[:ref]) || blank_rev?(change[:newrev])
[]
else
Gitlab::Lazy.new { commits_for(change[:oldrev], change[:newrev]) }
@@ -122,6 +122,14 @@ module Gitlab
def blank_rev?(rev)
rev.blank? || Gitlab::Git.blank_ref?(rev)
end
+
+ # refs/notes/commits contains commits added via `git-notes`. We currently
+ # have no features that check notes so we can skip them. To future-proof
+ # we are skipping anything that isn't a branch or tag ref as those are
+ # the only refs that can contain commits.
+ def commitish_ref?(ref)
+ Gitlab::Git.branch_ref?(ref) || Gitlab::Git.tag_ref?(ref)
+ end
end
end
end
diff --git a/lib/gitlab/checks/diff_check.rb b/lib/gitlab/checks/diff_check.rb
index d8f5cec8a4a..083c2448a0a 100644
--- a/lib/gitlab/checks/diff_check.rb
+++ b/lib/gitlab/checks/diff_check.rb
@@ -10,6 +10,10 @@ module Gitlab
}.freeze
def validate!
+ # git-notes stores notes history as commits in refs/notes/commits (by
+ # default but is configurable) so we restrict the diff checks to tag
+ # and branch refs
+ return unless tag_ref? || branch_ref?
return if deletion?
return unless should_run_validations?
return if commits.empty?
diff --git a/lib/gitlab/checks/matching_merge_request.rb b/lib/gitlab/checks/matching_merge_request.rb
index e5ce862264f..15178597a99 100644
--- a/lib/gitlab/checks/matching_merge_request.rb
+++ b/lib/gitlab/checks/matching_merge_request.rb
@@ -17,7 +17,7 @@ module Gitlab
#
# 1. Sidekiq: MergeService runs and updates the merge request in a locked state.
# 2. Gitaly: The UserMergeBranch RPC runs.
- # 3. Gitaly (gitaly-ruby): This RPC calls the pre-receive hook.
+ # 3. Gitaly: The RPC calls the pre-receive hook.
# 4. Rails: This hook makes an API request to /api/v4/internal/allowed.
# 5. Rails: This API check does a SQL query for locked merge
# requests with a matching SHA.
diff --git a/lib/gitlab/checks/single_change_access.rb b/lib/gitlab/checks/single_change_access.rb
index 2fd48dfbfe2..9f427e98e55 100644
--- a/lib/gitlab/checks/single_change_access.rb
+++ b/lib/gitlab/checks/single_change_access.rb
@@ -14,7 +14,9 @@ module Gitlab
protocol:, logger:, commits: nil
)
@oldrev, @newrev, @ref = change.values_at(:oldrev, :newrev, :ref)
+ @branch_ref = Gitlab::Git.branch_ref?(@ref)
@branch_name = Gitlab::Git.branch_name(@ref)
+ @tag_ref = Gitlab::Git.tag_ref?(@ref)
@tag_name = Gitlab::Git.tag_name(@ref)
@user_access = user_access
@project = project
@@ -38,6 +40,14 @@ module Gitlab
@commits ||= project.repository.new_commits(newrev)
end
+ def branch_ref?
+ @branch_ref
+ end
+
+ def tag_ref?
+ @tag_ref
+ end
+
protected
def ref_level_checks
diff --git a/lib/gitlab/ci/ansi2json/parser.rb b/lib/gitlab/ci/ansi2json/parser.rb
index fdd49df1e24..1d26bceb7b1 100644
--- a/lib/gitlab/ci/ansi2json/parser.rb
+++ b/lib/gitlab/ci/ansi2json/parser.rb
@@ -9,14 +9,14 @@ module Gitlab
class Parser
# keys represent the trailing digit in color changing command (30-37, 40-47, 90-97. 100-107)
COLOR = {
- 0 => 'black', # not that this is gray in the intense color table
+ 0 => 'black', # Note: This is gray in the intense color table.
1 => 'red',
2 => 'green',
3 => 'yellow',
4 => 'blue',
5 => 'magenta',
6 => 'cyan',
- 7 => 'white' # not that this is gray in the dark (aka default) color table
+ 7 => 'white' # Note: This is gray in the dark (aka default) color table.
}.freeze
STYLE_SWITCHES = {
diff --git a/lib/gitlab/ci/ansi2json/state.rb b/lib/gitlab/ci/ansi2json/state.rb
index b2b6ce649ed..3aec1cde1bc 100644
--- a/lib/gitlab/ci/ansi2json/state.rb
+++ b/lib/gitlab/ci/ansi2json/state.rb
@@ -1,11 +1,18 @@
# frozen_string_literal: true
+require 'openssl'
+
# In this class we keep track of the state changes that the
# Converter makes as it scans through the log stream.
module Gitlab
module Ci
module Ansi2json
class State
+ include Gitlab::Utils::StrongMemoize
+
+ SIGNATURE_KEY_SALT = 'gitlab-ci-ansi2json-state'
+ SEPARATOR = '--'
+
attr_accessor :offset, :current_line, :inherited_style, :open_sections, :last_line_offset
def initialize(new_state, stream_size)
@@ -18,12 +25,15 @@ module Gitlab
end
def encode
- state = {
+ json = {
offset: @last_line_offset,
style: @current_line.style.to_h,
open_sections: @open_sections
- }
- Base64.urlsafe_encode64(state.to_json)
+ }.to_json
+
+ encoded = Base64.urlsafe_encode64(json, padding: false)
+
+ encoded + SEPARATOR + sign(encoded)
end
def open_section(section, timestamp, options)
@@ -85,14 +95,55 @@ module Gitlab
end
end
- def decode_state(state)
- return unless state.present?
+ def decode_state(data)
+ return if data.blank?
- decoded_state = Base64.urlsafe_decode64(state)
+ encoded_state = verify(data)
+ if encoded_state.blank?
+ ::Gitlab::AppLogger.warn(message: "#{self.class}: signature missing or invalid", invalid_state: data)
+ return
+ end
+
+ decoded_state = Base64.urlsafe_decode64(encoded_state)
return unless decoded_state.present?
- Gitlab::Json.parse(decoded_state)
+ ::Gitlab::Json.parse(decoded_state)
+ end
+
+ def sign(message)
+ ::OpenSSL::HMAC.hexdigest(
+ signature_digest,
+ signature_key,
+ message
+ )
+ end
+
+ def verify(signed_message)
+ signature_length = signature_digest.digest_length * 2 # a byte is exactly two hexadecimals
+ message_length = signed_message.length - SEPARATOR.length - signature_length
+ return if message_length <= 0
+
+ signature = signed_message.last(signature_length)
+ message = signed_message.first(message_length)
+ return unless valid_signature?(message, signature)
+
+ message
+ end
+
+ def valid_signature?(message, signature)
+ expected_signature = sign(message)
+ expected_signature.bytesize == signature.bytesize &&
+ ::OpenSSL.fixed_length_secure_compare(signature, expected_signature)
+ end
+
+ def signature_digest
+ ::OpenSSL::Digest.new('SHA256')
+ end
+
+ def signature_key
+ ::Gitlab::Application.key_generator.generate_key(SIGNATURE_KEY_SALT, signature_digest.block_length)
end
+ strong_memoize_attr :signature_key
end
end
end
diff --git a/lib/gitlab/ci/badge/release/latest_release.rb b/lib/gitlab/ci/badge/release/latest_release.rb
index e73bb2a912a..8d84a54787b 100644
--- a/lib/gitlab/ci/badge/release/latest_release.rb
+++ b/lib/gitlab/ci/badge/release/latest_release.rb
@@ -10,7 +10,8 @@ module Gitlab::Ci
@project = project
@customization = {
key_width: opts[:key_width] ? opts[:key_width].to_i : nil,
- key_text: opts[:key_text]
+ key_text: opts[:key_text],
+ value_width: opts[:value_width] ? opts[:value_width].to_i : nil
}
# In the future, we should support `order_by=semver` for showing the
diff --git a/lib/gitlab/ci/badge/release/template.rb b/lib/gitlab/ci/badge/release/template.rb
index 354be6276fa..549742226a1 100644
--- a/lib/gitlab/ci/badge/release/template.rb
+++ b/lib/gitlab/ci/badge/release/template.rb
@@ -11,9 +11,11 @@ module Gitlab::Ci
}.freeze
KEY_WIDTH_DEFAULT = 90
VALUE_WIDTH_DEFAULT = 54
+ VALUE_WIDTH_MAXIMUM = 200
def initialize(badge)
@tag = badge.tag || "none"
+ @value_width = badge.customization[:value_width]
super
end
@@ -30,7 +32,11 @@ module Gitlab::Ci
end
def value_width
- VALUE_WIDTH_DEFAULT
+ if @value_width && @value_width.between?(1, VALUE_WIDTH_MAXIMUM)
+ @value_width
+ else
+ VALUE_WIDTH_DEFAULT
+ end
end
def value_color
diff --git a/lib/gitlab/ci/build/cache.rb b/lib/gitlab/ci/build/cache.rb
index 1cddc9fcc98..c1052f59272 100644
--- a/lib/gitlab/ci/build/cache.rb
+++ b/lib/gitlab/ci/build/cache.rb
@@ -9,8 +9,10 @@ module Gitlab
def initialize(cache, pipeline)
cache = Array.wrap(cache)
@cache = cache.map.with_index do |cache, index|
+ prefix = cache_prefix(cache, index)
+
Gitlab::Ci::Pipeline::Seed::Build::Cache
- .new(pipeline, cache, index)
+ .new(pipeline, cache, prefix)
end
end
@@ -23,6 +25,20 @@ module Gitlab
end
end
end
+
+ private
+
+ # The below method fixes a bug related to incorrect caches being used
+ # For more details please see: https://gitlab.com/gitlab-org/gitlab/-/issues/388374
+ def cache_prefix(cache, index)
+ files = cache.dig(:key, :files) if cache.is_a?(Hash) && cache[:key].is_a?(Hash)
+
+ return index if files.blank?
+
+ filenames = files.map { |file| file.split('.').first }.join('_')
+
+ "#{index}_#{filenames}"
+ end
end
end
end
diff --git a/lib/gitlab/ci/build/rules.rb b/lib/gitlab/ci/build/rules.rb
index dee95534b07..bc7aad1b186 100644
--- a/lib/gitlab/ci/build/rules.rb
+++ b/lib/gitlab/ci/build/rules.rb
@@ -6,12 +6,14 @@ module Gitlab
class Rules
include ::Gitlab::Utils::StrongMemoize
- Result = Struct.new(:when, :start_in, :allow_failure, :variables, :errors) do
+ Result = Struct.new(:when, :start_in, :allow_failure, :variables, :needs, :errors) do
def build_attributes
{
when: self.when,
options: { start_in: start_in }.compact,
- allow_failure: allow_failure
+ allow_failure: allow_failure,
+ scheduling_type: (:dag if needs),
+ needs_attributes: needs&.[](:job)
}.compact
end
@@ -33,13 +35,14 @@ module Gitlab
matched_rule.attributes[:when] || @default_when,
matched_rule.attributes[:start_in],
matched_rule.attributes[:allow_failure],
- matched_rule.attributes[:variables]
+ matched_rule.attributes[:variables],
+ (matched_rule.attributes[:needs] if Feature.enabled?(:introduce_rules_with_needs, pipeline.project))
)
else
Result.new('never')
end
rescue Rule::Clause::ParseError => e
- Result.new('never', nil, nil, nil, [e.message])
+ Result.new('never', nil, nil, nil, nil, [e.message])
end
private
diff --git a/lib/gitlab/ci/components/instance_path.rb b/lib/gitlab/ci/components/instance_path.rb
index 010ce57d2a0..27a7611ffdd 100644
--- a/lib/gitlab/ci/components/instance_path.rb
+++ b/lib/gitlab/ci/components/instance_path.rb
@@ -6,6 +6,8 @@ module Gitlab
class InstancePath
include Gitlab::Utils::StrongMemoize
+ LATEST_VERSION_KEYWORD = '~latest'
+
def self.match?(address)
address.include?('@') && address.start_with?(Settings.gitlab_ci['component_fqdn'])
end
@@ -39,9 +41,9 @@ module Gitlab
File.join(component_dir, @content_filename).delete_prefix('/')
end
- # TODO: Add support when version is a released tag and "~latest" moving target
def sha
return unless project
+ return latest_version_sha if version == LATEST_VERSION_KEYWORD
project.commit(version)&.id
end
@@ -69,6 +71,12 @@ module Gitlab
::Project.where_full_path_in(possible_paths).take # rubocop: disable CodeReuse/ActiveRecord
end
+
+ def latest_version_sha
+ return unless catalog_resource = project&.catalog_resource
+
+ catalog_resource.latest_version&.sha
+ end
end
end
end
diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb
index 585e671ce42..0c293c3f0ef 100644
--- a/lib/gitlab/ci/config.rb
+++ b/lib/gitlab/ci/config.rb
@@ -9,7 +9,7 @@ module Gitlab
include Gitlab::Utils::StrongMemoize
ConfigError = Class.new(StandardError)
- TIMEOUT_SECONDS = 30.seconds
+ TIMEOUT_SECONDS = ENV.fetch('GITLAB_CI_CONFIG_FETCH_TIMEOUT_SECONDS', 30).to_i.clamp(0, 60).seconds
TIMEOUT_MESSAGE = 'Request timed out when fetching configuration files.'
RESCUE_ERRORS = [
@@ -21,14 +21,15 @@ module Gitlab
attr_reader :root, :context, :source_ref_path, :source, :logger
- def initialize(config, project: nil, pipeline: nil, sha: nil, user: nil, parent_pipeline: nil, source: nil, logger: nil)
+ # rubocop: disable Metrics/ParameterLists
+ def initialize(config, project: nil, pipeline: nil, sha: nil, user: nil, parent_pipeline: nil, source: nil, pipeline_config: nil, logger: nil)
@logger = logger || ::Gitlab::Ci::Pipeline::Logger.new(project: project)
@source_ref_path = pipeline&.source_ref_path
@project = project
@context = self.logger.instrument(:config_build_context, once: true) do
pipeline ||= ::Ci::Pipeline.new(project: project, sha: sha, user: user, source: source)
- build_context(project: project, pipeline: pipeline, sha: sha, user: user, parent_pipeline: parent_pipeline)
+ build_context(project: project, pipeline: pipeline, sha: sha, user: user, parent_pipeline: parent_pipeline, pipeline_config: pipeline_config)
end
@context.set_deadline(TIMEOUT_SECONDS)
@@ -49,6 +50,7 @@ module Gitlab
rescue *rescue_errors => e
raise Config::ConfigError, e.message
end
+ # rubocop: enable Metrics/ParameterLists
def valid?
@root.valid?
@@ -117,8 +119,7 @@ module Gitlab
def expand_config(config)
build_config(config)
- rescue Gitlab::Config::Loader::Yaml::DataTooLargeError,
- Gitlab::Config::Loader::MultiDocYaml::DataTooLargeError => e
+ rescue Gitlab::Config::Loader::Yaml::DataTooLargeError => e
track_and_raise_for_dev_exception(e)
raise Config::ConfigError, e.message
@@ -157,13 +158,14 @@ module Gitlab
end
end
- def build_context(project:, pipeline:, sha:, user:, parent_pipeline:)
+ def build_context(project:, pipeline:, sha:, user:, parent_pipeline:, pipeline_config:)
Config::External::Context.new(
project: project,
sha: sha || find_sha(project),
user: user,
parent_pipeline: parent_pipeline,
variables: build_variables(pipeline: pipeline),
+ pipeline_config: pipeline_config,
logger: logger)
end
diff --git a/lib/gitlab/ci/config/entry/cache.rb b/lib/gitlab/ci/config/entry/cache.rb
index a635f409109..b3ff74c14da 100644
--- a/lib/gitlab/ci/config/entry/cache.rb
+++ b/lib/gitlab/ci/config/entry/cache.rb
@@ -9,11 +9,12 @@ module Gitlab
include ::Gitlab::Config::Entry::Validatable
include ::Gitlab::Config::Entry::Attributable
- ALLOWED_KEYS = %i[key untracked paths when policy unprotect].freeze
+ ALLOWED_KEYS = %i[key untracked paths when policy unprotect fallback_keys].freeze
ALLOWED_POLICY = %w[pull-push push pull].freeze
DEFAULT_POLICY = 'pull-push'
ALLOWED_WHEN = %w[on_success on_failure always].freeze
DEFAULT_WHEN = 'on_success'
+ DEFAULT_FALLBACK_KEYS = [].freeze
validations do
validates :config, type: Hash, allowed_keys: ALLOWED_KEYS
@@ -27,6 +28,8 @@ module Gitlab
in: ALLOWED_WHEN,
message: "should be one of: #{ALLOWED_WHEN.join(', ')}"
}
+
+ validates :fallback_keys, length: { maximum: 5, too_long: "has to many entries (maximum %{count})" }
end
end
@@ -42,7 +45,10 @@ module Gitlab
entry :paths, Entry::Paths,
description: 'Specify which paths should be cached across builds.'
- attributes :policy, :when, :unprotect
+ entry :fallback_keys, ::Gitlab::Config::Entry::ArrayOfStrings,
+ description: 'List of keys to download cache from if no cache hit occurred for key'
+
+ attributes :policy, :when, :unprotect, :fallback_keys
def value
result = super
@@ -52,6 +58,7 @@ module Gitlab
result[:policy] = policy || DEFAULT_POLICY
# Use self.when to avoid conflict with reserved word
result[:when] = self.when || DEFAULT_WHEN
+ result[:fallback_keys] = fallback_keys || DEFAULT_FALLBACK_KEYS
result
end
diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb
index 7c49b59a7f0..d31d1b366c3 100644
--- a/lib/gitlab/ci/config/entry/job.rb
+++ b/lib/gitlab/ci/config/entry/job.rb
@@ -14,7 +14,7 @@ module Gitlab
ALLOWED_KEYS = %i[tags script image services start_in artifacts
cache dependencies before_script after_script hooks
environment coverage retry parallel interruptible timeout
- release id_tokens].freeze
+ release id_tokens publish].freeze
validations do
validates :config, allowed_keys: Gitlab::Ci::Config::Entry::Job.allowed_keys + PROCESSABLE_ALLOWED_KEYS
@@ -45,6 +45,8 @@ module Gitlab
errors.add(:dependencies, "the #{missing_needs.join(", ")} should be part of needs") if missing_needs.any?
end
end
+
+ validates :publish, absence: { message: "can only be used within a `pages` job" }, unless: -> { pages_job? }
end
entry :before_script, Entry::Commands,
@@ -125,10 +127,14 @@ module Gitlab
inherit: false,
metadata: { composable_class: ::Gitlab::Ci::Config::Entry::IdToken }
+ entry :publish, Entry::Publish,
+ description: 'Path to be published with Pages',
+ inherit: false
+
attributes :script, :tags, :when, :dependencies,
:needs, :retry, :parallel, :start_in,
:interruptible, :timeout,
- :release, :allow_failure
+ :release, :allow_failure, :publish
def self.matching?(name, config)
!name.to_s.start_with?('.') &&
@@ -164,12 +170,13 @@ module Gitlab
artifacts: artifacts_value,
release: release_value,
after_script: after_script_value,
- hooks: hooks_pre_get_sources_script_enabled? ? hooks_value : nil,
+ hooks: hooks_value,
ignore: ignored?,
allow_failure_criteria: allow_failure_criteria,
needs: needs_defined? ? needs_value : nil,
scheduling_type: needs_defined? ? :dag : :stage,
- id_tokens: id_tokens_value
+ id_tokens: id_tokens_value,
+ publish: publish
).compact
end
@@ -177,6 +184,10 @@ module Gitlab
allow_failure_defined? ? static_allow_failure : manual_action?
end
+ def pages_job?
+ name == :pages
+ end
+
def self.allowed_keys
ALLOWED_KEYS
end
@@ -194,10 +205,6 @@ module Gitlab
allow_failure_value
end
-
- def hooks_pre_get_sources_script_enabled?
- YamlProcessor::FeatureFlags.enabled?(:ci_hooks_pre_get_sources_script)
- end
end
end
end
diff --git a/lib/gitlab/ci/config/entry/product/parallel.rb b/lib/gitlab/ci/config/entry/product/parallel.rb
index e91714e3f5c..59cd3d3cf91 100644
--- a/lib/gitlab/ci/config/entry/product/parallel.rb
+++ b/lib/gitlab/ci/config/entry/product/parallel.rb
@@ -19,7 +19,7 @@ module Gitlab
validations do
validates :config, numericality: { only_integer: true,
- greater_than_or_equal_to: 2,
+ greater_than_or_equal_to: 1,
less_than_or_equal_to: Entry::Product::Parallel::PARALLEL_LIMIT },
allow_nil: true
diff --git a/lib/gitlab/ci/config/entry/publish.rb b/lib/gitlab/ci/config/entry/publish.rb
new file mode 100644
index 00000000000..52a2487009e
--- /dev/null
+++ b/lib/gitlab/ci/config/entry/publish.rb
@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ class Config
+ module Entry
+ ##
+ # Entry that represents the path to be published with Pages.
+ #
+ class Publish < ::Gitlab::Config::Entry::Node
+ include ::Gitlab::Config::Entry::Validatable
+
+ validations do
+ validates :config, type: String
+ end
+
+ def self.default
+ 'public'
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/config/entry/rules/rule.rb b/lib/gitlab/ci/config/entry/rules/rule.rb
index 63bf1b38ac6..1e7f6056a65 100644
--- a/lib/gitlab/ci/config/entry/rules/rule.rb
+++ b/lib/gitlab/ci/config/entry/rules/rule.rb
@@ -9,7 +9,7 @@ module Gitlab
include ::Gitlab::Config::Entry::Configurable
include ::Gitlab::Config::Entry::Attributable
- ALLOWED_KEYS = %i[if changes exists when start_in allow_failure variables].freeze
+ ALLOWED_KEYS = %i[if changes exists when start_in allow_failure variables needs].freeze
ALLOWED_WHEN = %w[on_success on_failure always never manual delayed].freeze
attributes :if, :exists, :when, :start_in, :allow_failure
@@ -20,6 +20,11 @@ module Gitlab
entry :variables, Entry::Variables,
description: 'Environment variables to define for rule conditions.'
+ entry :needs, Entry::Needs,
+ description: 'Needs configuration to define for rule conditions.',
+ metadata: { allowed_needs: %i[job] },
+ inherit: false
+
validations do
validates :config, presence: true
validates :config, type: { with: Hash }
@@ -46,7 +51,8 @@ module Gitlab
def value
config.merge(
changes: (changes_value if changes_defined?),
- variables: (variables_value if variables_defined?)
+ variables: (variables_value if variables_defined?),
+ needs: (needs_value if needs_defined?)
).compact
end
diff --git a/lib/gitlab/ci/config/external/context.rb b/lib/gitlab/ci/config/external/context.rb
index 6eef279d3de..b8e012ec851 100644
--- a/lib/gitlab/ci/config/external/context.rb
+++ b/lib/gitlab/ci/config/external/context.rb
@@ -9,29 +9,29 @@ module Gitlab
TimeoutError = Class.new(StandardError)
- MAX_INCLUDES = 100
- NEW_MAX_INCLUDES = 150 # Update to MAX_INCLUDES when FF ci_includes_count_duplicates is removed
+ TEMP_MAX_INCLUDES = 100 # For logging; to be removed in https://gitlab.com/gitlab-org/gitlab/-/issues/396776
include ::Gitlab::Utils::StrongMemoize
- attr_reader :project, :sha, :user, :parent_pipeline, :variables
+ attr_reader :project, :sha, :user, :parent_pipeline, :variables, :pipeline_config
attr_reader :expandset, :execution_deadline, :logger, :max_includes
delegate :instrument, to: :logger
def initialize(
project: nil, sha: nil, user: nil, parent_pipeline: nil, variables: nil,
- logger: nil
+ pipeline_config: nil, logger: nil
)
@project = project
@sha = sha
@user = user
@parent_pipeline = parent_pipeline
@variables = variables || Ci::Variables::Collection.new
- @expandset = Feature.enabled?(:ci_includes_count_duplicates, project) ? [] : Set.new
+ @pipeline_config = pipeline_config
+ @expandset = []
@execution_deadline = 0
@logger = logger || Gitlab::Ci::Pipeline::Logger.new(project: project)
- @max_includes = Feature.enabled?(:ci_includes_count_duplicates, project) ? NEW_MAX_INCLUDES : MAX_INCLUDES
+ @max_includes = Gitlab::CurrentSettings.current_application_settings.ci_max_includes
yield self if block_given?
end
@@ -91,6 +91,13 @@ module Gitlab
expandset.map(&:metadata)
end
+ # Some Ci::ProjectConfig sources prepend the config content with an "internal" `include`, which becomes
+ # the first included file. When running a pipeline, we pass pipeline_config into the context of the first
+ # included file, which we use in this method to determine if the file is an "internal" one.
+ def internal_include?
+ !!pipeline_config&.internal_include_prepended?
+ end
+
protected
attr_writer :expandset, :execution_deadline, :logger, :max_includes
diff --git a/lib/gitlab/ci/config/external/file/artifact.rb b/lib/gitlab/ci/config/external/file/artifact.rb
index 0b90d240a15..273d78bd583 100644
--- a/lib/gitlab/ci/config/external/file/artifact.rb
+++ b/lib/gitlab/ci/config/external/file/artifact.rb
@@ -22,7 +22,7 @@ module Gitlab
strong_memoize(:content) do
Gitlab::Ci::ArtifactFileReader.new(artifact_job).read(location)
rescue Gitlab::Ci::ArtifactFileReader::Error => error
- errors.push(error.message)
+ errors.push(error.message) # TODO this memoizes the error message as a content!
end
end
diff --git a/lib/gitlab/ci/config/external/file/base.rb b/lib/gitlab/ci/config/external/file/base.rb
index 84f34f2584b..6b635cdf33b 100644
--- a/lib/gitlab/ci/config/external/file/base.rb
+++ b/lib/gitlab/ci/config/external/file/base.rb
@@ -61,16 +61,16 @@ module Gitlab
[params, context.project&.full_path, context.sha].hash
end
- def load_and_validate_expanded_hash!
- context.logger.instrument(:config_file_fetch_content_hash) do
- content_hash # calling the method loads then memoizes the result
- end
-
- context.logger.instrument(:config_file_expand_content_includes) do
- expanded_content_hash # calling the method expands then memoizes the result
- end
+ # This method is overridden to load context into the memoized result
+ # or to lazily load context via BatchLoader
+ def preload_context
+ # no-op
+ end
- validate_hash!
+ def preload_content
+ # calling the `content` method either loads content into the memoized result
+ # or lazily loads it via BatchLoader
+ content
end
def validate_location!
@@ -82,31 +82,65 @@ module Gitlab
end
def validate_context!
- raise NotImplementedError, 'subclass must implement validate_context'
+ raise NotImplementedError, 'subclass must implement `validate_context!`'
end
def validate_content!
- if content.blank?
- errors.push("Included file `#{masked_location}` is empty or does not exist!")
+ errors.push("Included file `#{masked_location}` is empty or does not exist!") if content.blank?
+ end
+
+ def load_and_validate_expanded_hash!
+ context.logger.instrument(:config_file_fetch_content_hash) do
+ content_result # calling the method loads YAML then memoizes the content result
+ end
+
+ context.logger.instrument(:config_file_interpolate_result) do
+ interpolator.interpolate!
+ end
+
+ return validate_interpolation! unless interpolator.valid?
+
+ context.logger.instrument(:config_file_expand_content_includes) do
+ expanded_content_hash # calling the method expands then memoizes the result
end
+
+ validate_hash!
end
protected
- def expanded_content_hash
- return unless content_hash
+ def content_result
+ ::Gitlab::Ci::Config::Yaml
+ .load_result!(content, project: context.project)
+ end
+ strong_memoize_attr :content_result
- strong_memoize(:expanded_content_hash) do
- expand_includes(content_hash)
- end
+ def content_inputs
+ # TODO: remove support for `with` syntax in 16.1, see https://gitlab.com/gitlab-org/gitlab/-/issues/408369
+ # In the interim prefer `inputs` over `with` while allow either syntax.
+ params.to_h.slice(:inputs, :with).each_value.first
end
+ strong_memoize_attr :content_inputs
def content_hash
- strong_memoize(:content_hash) do
- ::Gitlab::Ci::Config::Yaml.load!(content)
+ interpolator.interpolate!
+
+ interpolator.to_hash
+ end
+ strong_memoize_attr :content_hash
+
+ def interpolator
+ External::Interpolator
+ .new(content_result, content_inputs, context)
+ end
+ strong_memoize_attr :interpolator
+
+ def expanded_content_hash
+ return if content_hash.blank?
+
+ strong_memoize(:expanded_content_hash) do
+ expand_includes(content_hash)
end
- rescue Gitlab::Config::Loader::FormatError
- nil
end
def validate_hash!
@@ -115,6 +149,12 @@ module Gitlab
end
end
+ def validate_interpolation!
+ return if interpolator.valid?
+
+ errors.push("`#{masked_location}`: #{interpolator.error_message}")
+ end
+
def expand_includes(hash)
External::Processor.new(hash, context.mutate(expand_context_attrs)).perform
end
diff --git a/lib/gitlab/ci/config/external/file/component.rb b/lib/gitlab/ci/config/external/file/component.rb
index 33e7724bf9b..9679d78a1aa 100644
--- a/lib/gitlab/ci/config/external/file/component.rb
+++ b/lib/gitlab/ci/config/external/file/component.rb
@@ -11,11 +11,12 @@ module Gitlab
def initialize(params, context)
@location = params[:component]
+
super
end
def matching?
- super && ::Feature.enabled?(:ci_include_components, context.project)
+ super && ::Feature.enabled?(:ci_include_components, context.project&.root_namespace)
end
def content
@@ -48,9 +49,7 @@ module Gitlab
end
def validate_content!
- return if content.present?
-
- errors.push(component_result.message)
+ errors.push(component_result.message) unless content.present?
end
private
diff --git a/lib/gitlab/ci/config/external/file/project.rb b/lib/gitlab/ci/config/external/file/project.rb
index f8d4cb27710..16a6bc8a692 100644
--- a/lib/gitlab/ci/config/external/file/project.rb
+++ b/lib/gitlab/ci/config/external/file/project.rb
@@ -15,7 +15,8 @@ module Gitlab
# `Repository#blobs_at` does not support files with the `/` prefix.
@location = Gitlab::Utils.remove_leading_slashes(params[:file])
- @project_name = get_project_name(params[:project])
+ # We are using the same downcase in the `project` method.
+ @project_name = get_project_name(params[:project]).to_s.downcase
@ref_name = params[:ref] || 'HEAD'
super
@@ -39,6 +40,15 @@ module Gitlab
)
end
+ def preload_context
+ #
+ # calling these methods lazily loads them via BatchLoader
+ #
+ project
+ can_access_local_content?
+ sha
+ end
+
def validate_context!
if !can_access_local_content?
errors.push("Project `#{masked_project_name}` not found or access denied! Make sure any includes in the pipeline configuration are correctly defined.")
@@ -58,21 +68,55 @@ module Gitlab
private
def project
- strong_memoize(:project) do
- ::Project.find_by_full_path(project_name)
+ return legacy_project if ::Feature.disabled?(:ci_batch_project_includes_context, context.project)
+
+ # Although we use `where_full_path_in`, this BatchLoader does not reduce the number of queries to 1.
+ # That's because we use it in the `can_access_local_content?` and `sha` BatchLoaders
+ # as the `for` parameter. And this loads the project immediately.
+ BatchLoader.for(project_name)
+ .batch do |project_names, loader|
+ ::Project.where_full_path_in(project_names.uniq).each do |project|
+ # We are using the same downcase in the `initialize` method.
+ loader.call(project.full_path.downcase, project)
+ end
end
end
def can_access_local_content?
- strong_memoize(:can_access_local_content) do
- context.logger.instrument(:config_file_project_validate_access) do
- Ability.allowed?(context.user, :download_code, project)
+ if ::Feature.disabled?(:ci_batch_project_includes_context, context.project)
+ return legacy_can_access_local_content?
+ end
+
+ return if project.nil?
+
+ # We are force-loading the project with the `itself` method
+ # because the `project` variable can be a `BatchLoader` object and we should not
+ # pass a `BatchLoader` object in the `for` method to prevent unwanted behaviors.
+ BatchLoader.for(project.itself)
+ .batch(key: context.user) do |projects, loader, args|
+ projects.uniq.each do |project|
+ context.logger.instrument(:config_file_project_validate_access) do
+ loader.call(project, Ability.allowed?(args[:key], :download_code, project))
+ end
+ end
+ end
+ end
+
+ def sha
+ return legacy_sha if ::Feature.disabled?(:ci_batch_project_includes_context, context.project)
+ return if project.nil?
+
+ # with `itself`, we are force-loading the project
+ BatchLoader.for([project.itself, ref_name])
+ .batch do |project_ref_pairs, loader|
+ project_ref_pairs.uniq.each do |project, ref_name|
+ loader.call([project, ref_name], project.commit(ref_name).try(:sha))
end
end
end
def fetch_local_content
- BatchLoader.for([sha, location])
+ BatchLoader.for([sha.to_s, location])
.batch(key: project) do |locations, loader, args|
context.logger.instrument(:config_file_fetch_project_content) do
args[:key].repository.blobs_at(locations).each do |blob|
@@ -84,8 +128,22 @@ module Gitlab
end
end
- def sha
- strong_memoize(:sha) do
+ def legacy_project
+ strong_memoize(:legacy_project) do
+ ::Project.find_by_full_path(project_name)
+ end
+ end
+
+ def legacy_can_access_local_content?
+ strong_memoize(:legacy_can_access_local_content) do
+ context.logger.instrument(:config_file_project_validate_access) do
+ Ability.allowed?(context.user, :download_code, project)
+ end
+ end
+ end
+
+ def legacy_sha
+ strong_memoize(:legacy_sha) do
project.commit(ref_name).try(:sha)
end
end
@@ -94,7 +152,7 @@ module Gitlab
def expand_context_attrs
{
project: project,
- sha: sha,
+ sha: sha.to_s, # we need to use `.to_s` to load the value from the BatchLoader
user: context.user,
parent_pipeline: context.parent_pipeline,
variables: context.variables
diff --git a/lib/gitlab/ci/config/external/interpolator.rb b/lib/gitlab/ci/config/external/interpolator.rb
new file mode 100644
index 00000000000..f8af77fb246
--- /dev/null
+++ b/lib/gitlab/ci/config/external/interpolator.rb
@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ class Config
+ module External
+ ##
+ # Config::External::Interpolation perform includable file interpolation, and surfaces all possible interpolation
+ # errors. It is designed to provide an external file's validation context too.
+ #
+ class Interpolator
+ include ::Gitlab::Utils::StrongMemoize
+
+ attr_reader :config, :args, :ctx, :errors
+
+ def initialize(config, args, ctx = nil)
+ @config = config
+ @args = args.to_h
+ @ctx = ctx
+ @errors = []
+
+ validate!
+ end
+
+ def valid?
+ @errors.none?
+ end
+
+ def ready?
+ ##
+ # Interpolation is ready when it has been either interrupted by an error or finished with a result.
+ #
+ @result || @errors.any?
+ end
+
+ def interpolate?
+ enabled? && has_header? && valid?
+ end
+
+ def has_header?
+ config.has_header? && config.header.present?
+ end
+
+ def to_hash
+ @result.to_h
+ end
+
+ def error_message
+ # Interpolator can have multiple error messages, like: ["interpolation interrupted by errors", "unknown
+ # interpolation key: `abc`"] ?
+ #
+ # We are joining them together into a single one, because only one error can be surfaced when an external
+ # file gets included and is invalid. The limit to three error messages combined is more than required.
+ #
+ @errors.first(3).join(', ')
+ end
+
+ ##
+ # TODO Add `instrument.logger` instrumentation blocks:
+ # https://gitlab.com/gitlab-org/gitlab/-/issues/396722
+ #
+ def interpolate!
+ return {} unless valid?
+ return @result ||= content.to_h unless interpolate?
+
+ return @errors.concat(header.errors) unless header.valid?
+ return @errors.concat(inputs.errors) unless inputs.valid?
+ return @errors.concat(context.errors) unless context.valid?
+ return @errors.concat(template.errors) unless template.valid?
+
+ if ctx&.user
+ ::Gitlab::UsageDataCounters::HLLRedisCounter.track_event('ci_interpolation_users', values: ctx.user.id)
+ end
+
+ @result ||= template.interpolated.to_h.deep_symbolize_keys
+ end
+ strong_memoize_attr :interpolate!
+
+ private
+
+ def validate!
+ return errors.push('content does not have a valid YAML syntax') unless config.valid?
+
+ return unless has_header? && !enabled?
+
+ errors.push('can not evaluate included file because interpolation is disabled')
+ end
+
+ def enabled?
+ return false if ctx.nil?
+
+ ::Feature.enabled?(:ci_includable_files_interpolation, ctx.project)
+ end
+
+ def header
+ @entry ||= Ci::Config::Header::Root.new(config.header).tap do |header|
+ header.key = 'header'
+
+ header.compose!
+ end
+ end
+
+ def content
+ @content ||= config.content
+ end
+
+ def spec
+ @spec ||= header.inputs_value
+ end
+
+ def inputs
+ @inputs ||= Ci::Input::Inputs.new(spec, args)
+ end
+
+ def context
+ @context ||= Ci::Interpolation::Context.new({ inputs: inputs.to_hash })
+ end
+
+ def template
+ @template ||= ::Gitlab::Ci::Interpolation::Template
+ .new(content, context)
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/config/external/mapper/matcher.rb b/lib/gitlab/ci/config/external/mapper/matcher.rb
index e59eaa6d324..5072d0971cf 100644
--- a/lib/gitlab/ci/config/external/mapper/matcher.rb
+++ b/lib/gitlab/ci/config/external/mapper/matcher.rb
@@ -7,22 +7,13 @@ module Gitlab
class Mapper
# Matches the first file type that matches the given location
class Matcher < Base
- FILE_CLASSES = [
- External::File::Local,
- External::File::Project,
- External::File::Component,
- External::File::Remote,
- External::File::Template,
- External::File::Artifact
- ].freeze
-
- FILE_SUBKEYS = FILE_CLASSES.map { |f| f.name.demodulize.downcase }.freeze
+ include Gitlab::Utils::StrongMemoize
private
def process_without_instrumentation(locations)
locations.map do |location|
- matching = FILE_CLASSES.map do |file_class|
+ matching = file_classes.map do |file_class|
file_class.new(location, context)
end.select(&:matching?)
@@ -31,10 +22,10 @@ module Gitlab
elsif matching.empty?
raise Mapper::AmbigiousSpecificationError,
"`#{masked_location(location.to_json)}` does not have a valid subkey for include. " \
- "Valid subkeys are: `#{FILE_SUBKEYS.join('`, `')}`"
+ "Valid subkeys are: `#{file_subkeys.join('`, `')}`"
else
raise Mapper::AmbigiousSpecificationError,
- "Each include must use only one of: `#{FILE_SUBKEYS.join('`, `')}`"
+ "Each include must use only one of: `#{file_subkeys.join('`, `')}`"
end
end
end
@@ -42,6 +33,28 @@ module Gitlab
def masked_location(location)
context.mask_variables_from(location)
end
+
+ def file_subkeys
+ file_classes.map { |f| f.name.demodulize.downcase }.freeze
+ end
+ strong_memoize_attr :file_subkeys
+
+ def file_classes
+ classes = [
+ External::File::Local,
+ External::File::Project,
+ External::File::Remote,
+ External::File::Template,
+ External::File::Artifact
+ ]
+
+ if Feature.enabled?(:ci_include_components, context.project&.root_namespace)
+ classes << External::File::Component
+ end
+
+ classes
+ end
+ strong_memoize_attr :file_classes
end
end
end
diff --git a/lib/gitlab/ci/config/external/mapper/verifier.rb b/lib/gitlab/ci/config/external/mapper/verifier.rb
index 2982b0efb6c..3472f2c581a 100644
--- a/lib/gitlab/ci/config/external/mapper/verifier.rb
+++ b/lib/gitlab/ci/config/external/mapper/verifier.rb
@@ -9,8 +9,49 @@ module Gitlab
class Verifier < Base
private
+ # rubocop: disable Metrics/CyclomaticComplexity
def process_without_instrumentation(files)
+ if ::Feature.disabled?(:ci_batch_project_includes_context, context.project)
+ return legacy_process_without_instrumentation(files)
+ end
+
+ files.each do |file|
+ # When running a pipeline, some Ci::ProjectConfig sources prepend the config content with an
+ # "internal" `include`. We use this condition to exclude that `include` from the included file set.
+ context.expandset << file unless context.internal_include?
+ verify_max_includes!
+
+ verify_execution_time!
+
+ file.validate_location!
+ file.preload_context if file.valid?
+ end
+
+ # We do not combine the loops because we need to load the context of all files via `BatchLoader`.
+ files.each do |file| # rubocop:disable Style/CombinableLoops
+ verify_execution_time!
+
+ file.validate_context! if file.valid?
+ file.preload_content if file.valid?
+ end
+
+ # We do not combine the loops because we need to load the content of all files via `BatchLoader`.
+ files.each do |file| # rubocop:disable Style/CombinableLoops
+ verify_execution_time!
+
+ file.validate_content! if file.valid?
+ file.load_and_validate_expanded_hash! if file.valid?
+ end
+ end
+ # rubocop: enable Metrics/CyclomaticComplexity
+
+ def legacy_process_without_instrumentation(files)
files.each do |file|
+ # When running a pipeline, some Ci::ProjectConfig sources prepend the config content with an
+ # "internal" `include`. We use this condition to exclude that `include` from the included file set.
+ context.expandset << file unless context.internal_include?
+ verify_max_includes!
+
verify_execution_time!
file.validate_location!
@@ -21,23 +62,15 @@ module Gitlab
# We do not combine the loops because we need to load the content of all files before continuing
# to call `BatchLoader` for all locations.
files.each do |file| # rubocop:disable Style/CombinableLoops
- # Checking the max includes will be changed with https://gitlab.com/gitlab-org/gitlab/-/issues/367150
- verify_max_includes!
verify_execution_time!
file.validate_content! if file.valid?
file.load_and_validate_expanded_hash! if file.valid?
-
- if context.expandset.is_a?(Array) # To be removed when FF 'ci_includes_count_duplicates' is removed
- context.expandset << file
- else
- context.expandset.add(file)
- end
end
end
def verify_max_includes!
- return if context.expandset.count < context.max_includes
+ return if context.expandset.count <= context.max_includes
raise Mapper::TooManyIncludesError, "Maximum of #{context.max_includes} nested includes are allowed!"
end
diff --git a/lib/gitlab/ci/config/header/input.rb b/lib/gitlab/ci/config/header/input.rb
new file mode 100644
index 00000000000..7f0edaaac4c
--- /dev/null
+++ b/lib/gitlab/ci/config/header/input.rb
@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ class Config
+ module Header
+ ##
+ # Input parameter used for interpolation with the CI configuration.
+ #
+ class Input < ::Gitlab::Config::Entry::Node
+ include ::Gitlab::Config::Entry::Validatable
+ include ::Gitlab::Config::Entry::Attributable
+
+ attributes :default, prefix: :input
+
+ validations do
+ validates :config, type: Hash, allowed_keys: [:default]
+ validates :key, alphanumeric: true
+ validates :input_default, alphanumeric: true, allow_nil: true
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/config/header/root.rb b/lib/gitlab/ci/config/header/root.rb
new file mode 100644
index 00000000000..251682d13b4
--- /dev/null
+++ b/lib/gitlab/ci/config/header/root.rb
@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ class Config
+ module Header
+ ##
+ # This class represents the root entry of the GitLab CI configuration header.
+ #
+ # A header is the first document in a multi-doc YAML that contains metadata
+ # and specifications about the GitLab CI configuration (the second document).
+ #
+ # The header is optional. A CI configuration can also be represented with a
+ # YAML containing a single document.
+ class Root < ::Gitlab::Config::Entry::Node
+ include ::Gitlab::Config::Entry::Configurable
+
+ ALLOWED_KEYS = %i[spec].freeze
+
+ validations do
+ validates :config, type: Hash, allowed_keys: ALLOWED_KEYS
+ end
+
+ entry :spec, Header::Spec,
+ description: 'Specifications of the CI configuration.',
+ inherit: false,
+ default: {}
+
+ def inputs_value
+ spec_entry.inputs_value
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/config/header/spec.rb b/lib/gitlab/ci/config/header/spec.rb
new file mode 100644
index 00000000000..4753c1eb441
--- /dev/null
+++ b/lib/gitlab/ci/config/header/spec.rb
@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ class Config
+ module Header
+ class Spec < ::Gitlab::Config::Entry::Node
+ include ::Gitlab::Config::Entry::Configurable
+
+ ALLOWED_KEYS = %i[inputs].freeze
+
+ validations do
+ validates :config, allowed_keys: ALLOWED_KEYS
+ end
+
+ entry :inputs, ::Gitlab::Config::Entry::ComposableHash,
+ description: 'Allowed input parameters used for interpolation.',
+ inherit: false,
+ metadata: { composable_class: ::Gitlab::Ci::Config::Header::Input }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/config/yaml.rb b/lib/gitlab/ci/config/yaml.rb
index 94ef0afe7f9..729e7e3ac05 100644
--- a/lib/gitlab/ci/config/yaml.rb
+++ b/lib/gitlab/ci/config/yaml.rb
@@ -7,23 +7,39 @@ module Gitlab
AVAILABLE_TAGS = [Config::Yaml::Tags::Reference].freeze
MAX_DOCUMENTS = 2
- class << self
- def load!(content)
+ class Loader
+ def initialize(content, project: nil)
+ @content = content
+ @project = project
+ end
+
+ def load!
ensure_custom_tags
- if ::Feature.enabled?(:ci_multi_doc_yaml)
- Gitlab::Config::Loader::MultiDocYaml.new(
+ if project.present? && ::Feature.enabled?(:ci_multi_doc_yaml, project)
+ ::Gitlab::Config::Loader::MultiDocYaml.new(
content,
max_documents: MAX_DOCUMENTS,
- additional_permitted_classes: AVAILABLE_TAGS
- ).load!.first
+ additional_permitted_classes: AVAILABLE_TAGS,
+ reject_empty: true
+ ).load!
else
- Gitlab::Config::Loader::Yaml.new(content, additional_permitted_classes: AVAILABLE_TAGS).load!
+ ::Gitlab::Config::Loader::Yaml
+ .new(content, additional_permitted_classes: AVAILABLE_TAGS)
+ .load!
end
end
+ def to_result
+ Yaml::Result.new(config: load!, error: nil)
+ rescue ::Gitlab::Config::Loader::FormatError => e
+ Yaml::Result.new(error: e)
+ end
+
private
+ attr_reader :content, :project
+
def ensure_custom_tags
@ensure_custom_tags ||= begin
AVAILABLE_TAGS.each { |klass| Psych.add_tag(klass.tag, klass) }
@@ -32,6 +48,23 @@ module Gitlab
end
end
end
+
+ class << self
+ def load!(content, project: nil)
+ Loader.new(content, project: project).to_result.then do |result|
+ ##
+ # raise an error for backwards compatibility
+ #
+ raise result.error unless result.valid?
+
+ result.content
+ end
+ end
+
+ def load_result!(content, project: nil)
+ Loader.new(content, project: project).to_result
+ end
+ end
end
end
end
diff --git a/lib/gitlab/ci/config/yaml/result.rb b/lib/gitlab/ci/config/yaml/result.rb
new file mode 100644
index 00000000000..33f9a454106
--- /dev/null
+++ b/lib/gitlab/ci/config/yaml/result.rb
@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ class Config
+ module Yaml
+ class Result
+ attr_reader :error
+
+ def initialize(config: nil, error: nil)
+ @config = Array.wrap(config)
+ @error = error
+ end
+
+ def valid?
+ error.nil?
+ end
+
+ def has_header?
+ return false unless @config.first.is_a?(Hash)
+
+ @config.size > 1 && @config.first.key?(:spec)
+ end
+
+ def header
+ raise ArgumentError unless has_header?
+
+ @config.first
+ end
+
+ def content
+ return @config.last if has_header?
+
+ @config.first
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/input/arguments/base.rb b/lib/gitlab/ci/input/arguments/base.rb
new file mode 100644
index 00000000000..a46037c40ce
--- /dev/null
+++ b/lib/gitlab/ci/input/arguments/base.rb
@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ module Input
+ module Arguments
+ ##
+ # Input::Arguments::Base is a common abstraction for input arguments:
+ # - required
+ # - optional
+ # - with a default value
+ #
+ class Base
+ attr_reader :key, :value, :spec, :errors
+
+ ArgumentNotValidError = Class.new(StandardError)
+
+ def initialize(key, spec, value)
+ @key = key # hash key / argument name
+ @value = value # user-provided value
+ @spec = spec # configured specification
+ @errors = []
+
+ unless value.is_a?(String) || value.nil? # rubocop:disable Style/IfUnlessModifier
+ @errors.push("unsupported value in input argument `#{key}`")
+ end
+
+ validate!
+ end
+
+ def valid?
+ @errors.none?
+ end
+
+ def validate!
+ raise NotImplementedError
+ end
+
+ def to_value
+ raise NotImplementedError
+ end
+
+ def to_hash
+ raise ArgumentNotValidError unless valid?
+
+ @output ||= { key => to_value }
+ end
+
+ def self.matches?(spec)
+ raise NotImplementedError
+ end
+
+ private
+
+ def error(message)
+ @errors.push("`#{@key}` input: #{message}")
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/input/arguments/default.rb b/lib/gitlab/ci/input/arguments/default.rb
new file mode 100644
index 00000000000..c6762b04870
--- /dev/null
+++ b/lib/gitlab/ci/input/arguments/default.rb
@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ module Input
+ module Arguments
+ ##
+ # Input::Arguments::Default class represents user-provided input argument that has a default value.
+ #
+ class Default < Input::Arguments::Base
+ def validate!
+ return error('argument specification invalid') unless spec.key?(:default)
+
+ error('invalid default value') unless default.is_a?(String) || default.nil?
+ end
+
+ ##
+ # User-provided value needs to be specified, but it may be an empty string:
+ #
+ # ```yaml
+ # inputs:
+ # env:
+ # default: development
+ #
+ # with:
+ # env: ""
+ # ```
+ #
+ # The configuration above will result in `env` being an empty string.
+ #
+ def to_value
+ value.nil? ? default : value
+ end
+
+ def default
+ spec[:default]
+ end
+
+ def self.matches?(spec)
+ return false unless spec.is_a?(Hash)
+
+ spec.count == 1 && spec.each_key.first == :default
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/input/arguments/options.rb b/lib/gitlab/ci/input/arguments/options.rb
new file mode 100644
index 00000000000..855dab129be
--- /dev/null
+++ b/lib/gitlab/ci/input/arguments/options.rb
@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ module Input
+ module Arguments
+ ##
+ # Input::Arguments::Options class represents user-provided input argument that is an enum, and is only valid
+ # when the value provided is listed as an acceptable one.
+ #
+ class Options < Input::Arguments::Base
+ ##
+ # An empty value is valid if it is allowlisted:
+ #
+ # ```yaml
+ # inputs:
+ # run:
+ # - ""
+ # - tests
+ #
+ # with:
+ # run: ""
+ # ```
+ #
+ # The configuration above will return an empty value.
+ #
+ def validate!
+ return error('argument specification invalid') unless options.is_a?(Array)
+ return error('options argument empty') if options.empty?
+
+ if !value.nil?
+ error("argument value #{value} not allowlisted") unless options.include?(value)
+ else
+ error('argument not provided')
+ end
+ end
+
+ def to_value
+ value
+ end
+
+ def options
+ spec[:options]
+ end
+
+ def self.matches?(spec)
+ return false unless spec.is_a?(Hash)
+
+ spec.count == 1 && spec.each_key.first == :options
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/input/arguments/required.rb b/lib/gitlab/ci/input/arguments/required.rb
new file mode 100644
index 00000000000..2e39f548731
--- /dev/null
+++ b/lib/gitlab/ci/input/arguments/required.rb
@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ module Input
+ module Arguments
+ ##
+ # Input::Arguments::Required class represents user-provided required input argument.
+ #
+ class Required < Input::Arguments::Base
+ ##
+ # The value has to be defined, but it may be empty.
+ #
+ def validate!
+ error('required value has not been provided') if value.nil?
+ end
+
+ def to_value
+ value
+ end
+
+ ##
+ # Required arguments do not have nested configuration. It has to be defined a null value.
+ #
+ # ```yaml
+ # spec:
+ # inputs:
+ # website:
+ # ```
+ #
+ # An empty string value, that has no specification is also considered as a "required" input, however we should
+ # never see that being used, because it will be rejected by Ci::Config::Header validation.
+ #
+ # ```yaml
+ # spec:
+ # inputs:
+ # website: ""
+ # ```
+ #
+ # An empty hash value is also considered to be a required argument:
+ #
+ # ```yaml
+ # spec:
+ # inputs:
+ # website: {}
+ # ```
+ #
+ def self.matches?(spec)
+ spec.blank?
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/input/arguments/unknown.rb b/lib/gitlab/ci/input/arguments/unknown.rb
new file mode 100644
index 00000000000..5873e6e66a6
--- /dev/null
+++ b/lib/gitlab/ci/input/arguments/unknown.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ module Input
+ module Arguments
+ ##
+ # Input::Arguments::Unknown object gets fabricated when we can't match an input argument entry with any known
+ # specification. It is matched as the last one, and always returns an error.
+ #
+ class Unknown < Input::Arguments::Base
+ def validate!
+ if spec.is_a?(Hash) && spec.count == 1
+ error("unrecognized input argument specification: `#{spec.each_key.first}`")
+ else
+ error('unrecognized input argument definition')
+ end
+ end
+
+ def to_value
+ raise ArgumentError, 'unknown argument value'
+ end
+
+ def self.matches?(*)
+ true
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/input/inputs.rb b/lib/gitlab/ci/input/inputs.rb
new file mode 100644
index 00000000000..1b544e63e7d
--- /dev/null
+++ b/lib/gitlab/ci/input/inputs.rb
@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ module Input
+ ##
+ # Inputs::Input class represents user-provided inputs, configured using `with:` keyword.
+ #
+ # Input arguments are only valid with an associated component's inputs specification from component's header.
+ #
+ class Inputs
+ UnknownSpecArgumentError = Class.new(StandardError)
+
+ ARGUMENTS = [
+ Input::Arguments::Required, # Input argument is required
+ Input::Arguments::Default, # Input argument has a default value
+ Input::Arguments::Options, # Input argument that needs to be allowlisted
+ Input::Arguments::Unknown # Input argument has not been recognized
+ ].freeze
+
+ def initialize(spec, args)
+ @spec = spec.to_h
+ @args = args.to_h
+ @inputs = []
+ @errors = []
+
+ validate!
+ fabricate!
+ end
+
+ def errors
+ @errors + @inputs.flat_map(&:errors)
+ end
+
+ def valid?
+ errors.none?
+ end
+
+ def unknown
+ @args.keys - @spec.keys
+ end
+
+ def count
+ @inputs.count
+ end
+
+ def to_hash
+ @inputs.inject({}) do |hash, argument|
+ raise ArgumentError unless argument.valid?
+
+ hash.merge(argument.to_hash)
+ end
+ end
+
+ private
+
+ def validate!
+ @errors.push("unknown input arguments: #{unknown.inspect}") if unknown.any?
+ end
+
+ def fabricate!
+ @spec.each do |key, spec|
+ argument = ARGUMENTS.find { |klass| klass.matches?(spec) }
+
+ raise UnknownSpecArgumentError if argument.nil?
+
+ @inputs.push(argument.new(key, spec, @args[key]))
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/interpolation/access.rb b/lib/gitlab/ci/interpolation/access.rb
index 42598458902..f9bbd3e118d 100644
--- a/lib/gitlab/ci/interpolation/access.rb
+++ b/lib/gitlab/ci/interpolation/access.rb
@@ -45,7 +45,11 @@ module Gitlab
raise ArgumentError, 'access path invalid' unless valid?
@value ||= objects.inject(@ctx) do |memo, value|
- memo.fetch(value.to_sym)
+ key = value.to_sym
+
+ break @errors.push("unknown interpolation key: `#{key}`") unless memo.key?(key)
+
+ memo.fetch(key)
end
rescue KeyError => e
@errors.push(e)
diff --git a/lib/gitlab/ci/interpolation/context.rb b/lib/gitlab/ci/interpolation/context.rb
index ce7a86a3c9b..69c1fbb792c 100644
--- a/lib/gitlab/ci/interpolation/context.rb
+++ b/lib/gitlab/ci/interpolation/context.rb
@@ -38,6 +38,10 @@ module Gitlab
@context.fetch(field)
end
+ def key?(name)
+ @context.key?(name)
+ end
+
def to_h
@context.to_h
end
@@ -53,7 +57,7 @@ module Gitlab
end
end
- values.max
+ values.max.to_i
end
def self.fabricate(context)
diff --git a/lib/gitlab/ci/jwt.rb b/lib/gitlab/ci/jwt.rb
index d82ca875e76..4ba7b4cc6e1 100644
--- a/lib/gitlab/ci/jwt.rb
+++ b/lib/gitlab/ci/jwt.rb
@@ -31,6 +31,11 @@ module Gitlab
attr_reader :build, :ttl
+ delegate :project, :user, :pipeline, :runner, to: :build
+ delegate :source_ref, :source_ref_path, to: :pipeline
+ delegate :public_key, to: :key
+ delegate :namespace, to: :project
+
def reserved_claims
now = Time.now.to_i
@@ -53,11 +58,12 @@ module Gitlab
user_id: user&.id.to_s,
user_login: user&.username,
user_email: user&.email,
- pipeline_id: build.pipeline.id.to_s,
- pipeline_source: build.pipeline.source.to_s,
+ pipeline_id: pipeline.id.to_s,
+ pipeline_source: pipeline.source.to_s,
job_id: build.id.to_s,
ref: source_ref,
ref_type: ref_type,
+ ref_path: source_ref_path,
ref_protected: build.protected.to_s
}
@@ -82,30 +88,10 @@ module Gitlab
end
end
- def public_key
- key.public_key
- end
-
def kid
public_key.to_jwk[:kid]
end
- def project
- build.project
- end
-
- def namespace
- project.namespace
- end
-
- def user
- build.user
- end
-
- def source_ref
- build.pipeline.source_ref
- end
-
def ref_type
::Ci::BuildRunnerPresenter.new(build).ref_type
end
diff --git a/lib/gitlab/ci/jwt_v2.rb b/lib/gitlab/ci/jwt_v2.rb
index cfefa79d9e0..aff30455d09 100644
--- a/lib/gitlab/ci/jwt_v2.rb
+++ b/lib/gitlab/ci/jwt_v2.rb
@@ -4,6 +4,8 @@ module Gitlab
module Ci
class JwtV2 < Jwt
DEFAULT_AUD = Settings.gitlab.base_url
+ GITLAB_HOSTED_RUNNER = 'gitlab-hosted'
+ SELF_HOSTED_RUNNER = 'self-hosted'
def self.for_build(build, aud: DEFAULT_AUD)
new(build, ttl: build.metadata_timeout, aud: aud).encoded
@@ -20,12 +22,38 @@ module Gitlab
attr_reader :aud
def reserved_claims
- super.merge(
+ super.merge({
iss: Settings.gitlab.base_url,
sub: "project_path:#{project.full_path}:ref_type:#{ref_type}:ref:#{source_ref}",
- aud: aud
+ aud: aud,
+ user_identities: user_identities
+ }.compact)
+ end
+
+ def user_identities
+ return unless user&.pass_user_identities_to_ci_jwt
+
+ user.identities.map do |identity|
+ {
+ provider: identity.provider.to_s,
+ extern_uid: identity.extern_uid.to_s
+ }
+ end
+ end
+
+ def custom_claims
+ super.merge(
+ runner_id: runner&.id,
+ runner_environment: runner_environment,
+ sha: pipeline.sha
)
end
+
+ def runner_environment
+ return unless runner
+
+ runner.gitlab_hosted? ? GITLAB_HOSTED_RUNNER : SELF_HOSTED_RUNNER
+ end
end
end
end
diff --git a/lib/gitlab/ci/parsers/security/common.rb b/lib/gitlab/ci/parsers/security/common.rb
index 1b9afc92d6b..447136df81f 100644
--- a/lib/gitlab/ci/parsers/security/common.rb
+++ b/lib/gitlab/ci/parsers/security/common.rb
@@ -139,6 +139,7 @@ module Gitlab
details: data['details'] || {},
signatures: signatures,
project_id: @project.id,
+ found_by_pipeline: report.pipeline,
vulnerability_finding_signatures_enabled: @signatures_enabled))
end
diff --git a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb
index bef4b147359..92d9d170575 100644
--- a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb
+++ b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb
@@ -7,27 +7,27 @@ module Gitlab
module Validators
class SchemaValidator
SUPPORTED_VERSIONS = {
- cluster_image_scanning: %w[14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4],
- container_scanning: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4],
- coverage_fuzzing: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4],
- dast: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4],
- api_fuzzing: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4],
- dependency_scanning: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4],
- sast: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4],
- secret_detection: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4]
+ cluster_image_scanning: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.6],
+ container_scanning: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.6],
+ coverage_fuzzing: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.6],
+ dast: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.6],
+ api_fuzzing: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.6],
+ dependency_scanning: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.6],
+ sast: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.6],
+ secret_detection: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.6]
}.freeze
- VERSIONS_TO_REMOVE_IN_16_0 = %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3].freeze
+ VERSIONS_TO_REMOVE_IN_17_0 = %w[].freeze
DEPRECATED_VERSIONS = {
- cluster_image_scanning: VERSIONS_TO_REMOVE_IN_16_0,
- container_scanning: VERSIONS_TO_REMOVE_IN_16_0,
- coverage_fuzzing: VERSIONS_TO_REMOVE_IN_16_0,
- dast: VERSIONS_TO_REMOVE_IN_16_0,
- api_fuzzing: VERSIONS_TO_REMOVE_IN_16_0,
- dependency_scanning: VERSIONS_TO_REMOVE_IN_16_0,
- sast: VERSIONS_TO_REMOVE_IN_16_0,
- secret_detection: VERSIONS_TO_REMOVE_IN_16_0
+ cluster_image_scanning: VERSIONS_TO_REMOVE_IN_17_0,
+ container_scanning: VERSIONS_TO_REMOVE_IN_17_0,
+ coverage_fuzzing: VERSIONS_TO_REMOVE_IN_17_0,
+ dast: VERSIONS_TO_REMOVE_IN_17_0,
+ api_fuzzing: VERSIONS_TO_REMOVE_IN_17_0,
+ dependency_scanning: VERSIONS_TO_REMOVE_IN_17_0,
+ sast: VERSIONS_TO_REMOVE_IN_17_0,
+ secret_detection: VERSIONS_TO_REMOVE_IN_17_0
}.freeze
CURRENT_VERSIONS = SUPPORTED_VERSIONS.to_h { |k, v| [k, v - DEPRECATED_VERSIONS[k]] }
@@ -131,11 +131,6 @@ module Gitlab
end
def report_uses_deprecated_schema_version?
- # Avoid deprecation warnings for GitLab security scanners
- # To be removed via https://gitlab.com/gitlab-org/gitlab/-/issues/386798
- return if report_data.dig('scan', 'scanner', 'vendor', 'name')&.downcase == 'gitlab'
- return if report_data.dig('scan', 'analyzer', 'vendor', 'name')&.downcase == 'gitlab'
-
DEPRECATED_VERSIONS[report_type].include?(report_version)
end
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.0/container-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.0/container-scanning-report-format.json
deleted file mode 100644
index 14eb376485f..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.0/container-scanning-report-format.json
+++ /dev/null
@@ -1,741 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Report format for GitLab Container Scanning",
- "description": "This schema provides the the report format for Container Scanning (https://docs.gitlab.com/ee/user/application_security/container_scanning).",
- "definitions": {
- "detail_type": {
- "oneOf": [
- {
- "$ref": "#/definitions/named_list"
- },
- {
- "$ref": "#/definitions/list"
- },
- {
- "$ref": "#/definitions/table"
- },
- {
- "$ref": "#/definitions/text"
- },
- {
- "$ref": "#/definitions/url"
- },
- {
- "$ref": "#/definitions/code"
- },
- {
- "$ref": "#/definitions/value"
- },
- {
- "$ref": "#/definitions/diff"
- },
- {
- "$ref": "#/definitions/markdown"
- },
- {
- "$ref": "#/definitions/commit"
- },
- {
- "$ref": "#/definitions/file_location"
- },
- {
- "$ref": "#/definitions/module_location"
- }
- ]
- },
- "text_value": {
- "type": "string"
- },
- "named_field": {
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "$ref": "#/definitions/text_value",
- "minLength": 1
- },
- "description": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "named_list": {
- "type": "object",
- "description": "An object with named and typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "named-list"
- },
- "items": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- {
- "$ref": "#/definitions/named_field"
- },
- {
- "$ref": "#/definitions/detail_type"
- }
- ]
- }
- }
- }
- }
- },
- "list": {
- "type": "object",
- "description": "A list of typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- },
- "table": {
- "type": "object",
- "description": "A table of typed fields",
- "required": [
- "type",
- "rows"
- ],
- "properties": {
- "type": {
- "const": "table"
- },
- "header": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- },
- "rows": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- }
- },
- "text": {
- "type": "object",
- "description": "Raw text",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "text"
- },
- "value": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "url": {
- "type": "object",
- "description": "A single URL",
- "required": [
- "type",
- "href"
- ],
- "properties": {
- "type": {
- "const": "url"
- },
- "text": {
- "$ref": "#/definitions/text_value"
- },
- "href": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "http://mysite.com"
- ]
- }
- }
- },
- "code": {
- "type": "object",
- "description": "A codeblock",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "code"
- },
- "value": {
- "type": "string"
- },
- "lang": {
- "type": "string",
- "description": "A programming language"
- }
- }
- },
- "value": {
- "type": "object",
- "description": "A field that can store a range of types of value",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "value"
- },
- "value": {
- "type": [
- "number",
- "string",
- "boolean"
- ]
- }
- }
- },
- "diff": {
- "type": "object",
- "description": "A diff",
- "required": [
- "type",
- "before",
- "after"
- ],
- "properties": {
- "type": {
- "const": "diff"
- },
- "before": {
- "type": "string"
- },
- "after": {
- "type": "string"
- }
- }
- },
- "markdown": {
- "type": "object",
- "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "markdown"
- },
- "value": {
- "$ref": "#/definitions/text_value",
- "examples": [
- "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
- ]
- }
- }
- },
- "commit": {
- "type": "object",
- "description": "A commit/tag/branch within the GitLab project",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "commit"
- },
- "value": {
- "type": "string",
- "description": "The commit SHA",
- "minLength": 1
- }
- }
- },
- "file_location": {
- "type": "object",
- "description": "A location within a file in the project",
- "required": [
- "type",
- "file_name",
- "line_start"
- ],
- "properties": {
- "type": {
- "const": "file-location"
- },
- "file_name": {
- "type": "string",
- "minLength": 1
- },
- "line_start": {
- "type": "integer"
- },
- "line_end": {
- "type": "integer"
- }
- }
- },
- "module_location": {
- "type": "object",
- "description": "A location within a binary module of the form module+relative_offset",
- "required": [
- "type",
- "module_name",
- "offset"
- ],
- "properties": {
- "type": {
- "const": "module-location"
- },
- "module_name": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "compiled_binary"
- ]
- },
- "offset": {
- "type": "integer",
- "examples": [
- 100
- ]
- }
- }
- }
- },
- "self": {
- "version": "14.0.0"
- },
- "required": [
- "version",
- "vulnerabilities"
- ],
- "additionalProperties": true,
- "properties": {
- "scan": {
- "type": "object",
- "required": [
- "end_time",
- "scanner",
- "start_time",
- "status",
- "type"
- ],
- "properties": {
- "end_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-01-28T03:26:02"
- ]
- },
- "messages": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Communication intended for the initiator of a scan.",
- "required": [
- "level",
- "value"
- ],
- "properties": {
- "level": {
- "type": "string",
- "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
- "enum": [
- "info",
- "warn",
- "fatal"
- ],
- "examples": [
- "info"
- ]
- },
- "value": {
- "type": "string",
- "description": "The message to communicate.",
- "minLength": 1,
- "examples": [
- "Permission denied, scanning aborted"
- ]
- }
- }
- }
- },
- "scanner": {
- "type": "object",
- "description": "Object defining the scanner used to perform the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the scanner.",
- "minLength": 1,
- "examples": [
- "my-sast-scanner"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the scanner, not required to be unique.",
- "minLength": 1,
- "examples": [
- "My SAST Scanner"
- ]
- },
- "url": {
- "type": "string",
- "description": "A link to more information about the scanner.",
- "examples": [
- "https://scanner.url"
- ]
- },
- "version": {
- "type": "string",
- "description": "The version of the scanner.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- },
- "vendor": {
- "type": "object",
- "description": "The vendor/maintainer of the scanner.",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- }
- }
- },
- "start_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-02-14T16:01:59"
- ]
- },
- "status": {
- "type": "string",
- "description": "Result of the scan.",
- "enum": [
- "success",
- "failure"
- ]
- },
- "type": {
- "type": "string",
- "description": "Type of the scan.",
- "enum": [
- "container_scanning"
- ]
- }
- }
- },
- "schema": {
- "type": "string",
- "description": "URI pointing to the validating security report schema.",
- "format": "uri"
- },
- "version": {
- "type": "string",
- "description": "The version of the schema to which the JSON report conforms.",
- "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
- },
- "vulnerabilities": {
- "type": "array",
- "description": "Array of vulnerability objects.",
- "items": {
- "type": "object",
- "description": "Describes the vulnerability.",
- "required": [
- "category",
- "cve",
- "identifiers",
- "location",
- "scanner"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
- "examples": [
- "642735a5-1425-428d-8d4e-3c854885a3c9"
- ]
- },
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
- "name": {
- "type": "string",
- "description": "The name of the vulnerability. This must not include the finding's specific information."
- },
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
- "description": {
- "type": "string",
- "description": "A long text section describing the vulnerability more fully."
- },
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
- "severity": {
- "type": "string",
- "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Info",
- "Unknown",
- "Low",
- "Medium",
- "High",
- "Critical"
- ]
- },
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
- "solution": {
- "type": "string",
- "description": "Explanation of how to fix the vulnerability."
- },
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
- "identifiers": {
- "type": "array",
- "minItems": 1,
- "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
- "items": {
- "type": "object",
- "required": [
- "type",
- "name",
- "value"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
- "minLength": 1
- },
- "name": {
- "type": "string",
- "description": "Human-readable name of the identifier.",
- "minLength": 1
- },
- "url": {
- "type": "string",
- "description": "URL of the identifier's documentation.",
- "format": "uri"
- },
- "value": {
- "type": "string",
- "description": "Value of the identifier, for matching purpose.",
- "minLength": 1
- }
- }
- }
- },
- "links": {
- "type": "array",
- "description": "An array of references to external documentation or articles that describe the vulnerability.",
- "items": {
- "type": "object",
- "required": [
- "url"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the vulnerability details link."
- },
- "url": {
- "type": "string",
- "description": "URL of the vulnerability details document.",
- "format": "uri"
- }
- }
- }
- },
- "details": {
- "$ref": "#/definitions/named_list/properties/items"
- },
- "location": {
- "type": "object",
- "description": "Identifies the vulnerability's location.",
- "required": [
- "dependency",
- "operating_system",
- "image"
- ],
- "properties": {
- "dependency": {
- "type": "object",
- "description": "Describes the dependency of a project where the vulnerability is located.",
- "properties": {
- "package": {
- "type": "object",
- "description": "Provides information on the package where the vulnerability is located.",
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the package where the vulnerability is located."
- }
- }
- },
- "version": {
- "type": "string",
- "description": "Version of the vulnerable package."
- },
- "iid": {
- "description": "ID that identifies the dependency in the scope of a dependency file.",
- "type": "number"
- },
- "direct": {
- "type": "boolean",
- "description": "Tells whether this is a direct, top-level dependency of the scanned project."
- },
- "dependency_path": {
- "type": "array",
- "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.",
- "items": {
- "type": "object",
- "required": [
- "iid"
- ],
- "properties": {
- "iid": {
- "type": "number",
- "description": "ID that is unique in the scope of a parent object, and specific to the resource type."
- }
- }
- }
- }
- }
- },
- "operating_system": {
- "type": "string",
- "minLength": 1,
- "description": "The operating system that contains the vulnerable package."
- },
- "image": {
- "type": "string",
- "minLength": 1,
- "description": "The analyzed Docker image."
- }
- }
- }
- }
- }
- },
- "remediations": {
- "type": "array",
- "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
- "items": {
- "type": "object",
- "required": [
- "fixes",
- "summary",
- "diff"
- ],
- "properties": {
- "fixes": {
- "type": "array",
- "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
- "items": {
- "type": "object",
- "required": [
- "cve"
- ],
- "properties": {
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- }
- }
- }
- },
- "summary": {
- "type": "string",
- "minLength": 1,
- "description": "An overview of how the vulnerabilities were fixed."
- },
- "diff": {
- "type": "string",
- "minLength": 1,
- "description": "A base64-encoded remediation code diff, compatible with git apply."
- }
- }
- }
- }
- }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.0/coverage-fuzzing-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.0/coverage-fuzzing-report-format.json
deleted file mode 100644
index 296a895c7cb..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.0/coverage-fuzzing-report-format.json
+++ /dev/null
@@ -1,711 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Report format for GitLab Fuzz Testing",
- "description": "This schema provides the report format for Coverage Guided Fuzz Testing (https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing).",
- "definitions": {
- "detail_type": {
- "oneOf": [
- {
- "$ref": "#/definitions/named_list"
- },
- {
- "$ref": "#/definitions/list"
- },
- {
- "$ref": "#/definitions/table"
- },
- {
- "$ref": "#/definitions/text"
- },
- {
- "$ref": "#/definitions/url"
- },
- {
- "$ref": "#/definitions/code"
- },
- {
- "$ref": "#/definitions/value"
- },
- {
- "$ref": "#/definitions/diff"
- },
- {
- "$ref": "#/definitions/markdown"
- },
- {
- "$ref": "#/definitions/commit"
- },
- {
- "$ref": "#/definitions/file_location"
- },
- {
- "$ref": "#/definitions/module_location"
- }
- ]
- },
- "text_value": {
- "type": "string"
- },
- "named_field": {
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "$ref": "#/definitions/text_value",
- "minLength": 1
- },
- "description": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "named_list": {
- "type": "object",
- "description": "An object with named and typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "named-list"
- },
- "items": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- {
- "$ref": "#/definitions/named_field"
- },
- {
- "$ref": "#/definitions/detail_type"
- }
- ]
- }
- }
- }
- }
- },
- "list": {
- "type": "object",
- "description": "A list of typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- },
- "table": {
- "type": "object",
- "description": "A table of typed fields",
- "required": [
- "type",
- "rows"
- ],
- "properties": {
- "type": {
- "const": "table"
- },
- "header": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- },
- "rows": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- }
- },
- "text": {
- "type": "object",
- "description": "Raw text",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "text"
- },
- "value": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "url": {
- "type": "object",
- "description": "A single URL",
- "required": [
- "type",
- "href"
- ],
- "properties": {
- "type": {
- "const": "url"
- },
- "text": {
- "$ref": "#/definitions/text_value"
- },
- "href": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "http://mysite.com"
- ]
- }
- }
- },
- "code": {
- "type": "object",
- "description": "A codeblock",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "code"
- },
- "value": {
- "type": "string"
- },
- "lang": {
- "type": "string",
- "description": "A programming language"
- }
- }
- },
- "value": {
- "type": "object",
- "description": "A field that can store a range of types of value",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "value"
- },
- "value": {
- "type": [
- "number",
- "string",
- "boolean"
- ]
- }
- }
- },
- "diff": {
- "type": "object",
- "description": "A diff",
- "required": [
- "type",
- "before",
- "after"
- ],
- "properties": {
- "type": {
- "const": "diff"
- },
- "before": {
- "type": "string"
- },
- "after": {
- "type": "string"
- }
- }
- },
- "markdown": {
- "type": "object",
- "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "markdown"
- },
- "value": {
- "$ref": "#/definitions/text_value",
- "examples": [
- "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
- ]
- }
- }
- },
- "commit": {
- "type": "object",
- "description": "A commit/tag/branch within the GitLab project",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "commit"
- },
- "value": {
- "type": "string",
- "description": "The commit SHA",
- "minLength": 1
- }
- }
- },
- "file_location": {
- "type": "object",
- "description": "A location within a file in the project",
- "required": [
- "type",
- "file_name",
- "line_start"
- ],
- "properties": {
- "type": {
- "const": "file-location"
- },
- "file_name": {
- "type": "string",
- "minLength": 1
- },
- "line_start": {
- "type": "integer"
- },
- "line_end": {
- "type": "integer"
- }
- }
- },
- "module_location": {
- "type": "object",
- "description": "A location within a binary module of the form module+relative_offset",
- "required": [
- "type",
- "module_name",
- "offset"
- ],
- "properties": {
- "type": {
- "const": "module-location"
- },
- "module_name": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "compiled_binary"
- ]
- },
- "offset": {
- "type": "integer",
- "examples": [
- 100
- ]
- }
- }
- }
- },
- "self": {
- "version": "14.0.0"
- },
- "required": [
- "version",
- "vulnerabilities"
- ],
- "additionalProperties": true,
- "properties": {
- "scan": {
- "type": "object",
- "required": [
- "end_time",
- "scanner",
- "start_time",
- "status",
- "type"
- ],
- "properties": {
- "end_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-01-28T03:26:02"
- ]
- },
- "messages": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Communication intended for the initiator of a scan.",
- "required": [
- "level",
- "value"
- ],
- "properties": {
- "level": {
- "type": "string",
- "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
- "enum": [
- "info",
- "warn",
- "fatal"
- ],
- "examples": [
- "info"
- ]
- },
- "value": {
- "type": "string",
- "description": "The message to communicate.",
- "minLength": 1,
- "examples": [
- "Permission denied, scanning aborted"
- ]
- }
- }
- }
- },
- "scanner": {
- "type": "object",
- "description": "Object defining the scanner used to perform the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the scanner.",
- "minLength": 1,
- "examples": [
- "my-sast-scanner"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the scanner, not required to be unique.",
- "minLength": 1,
- "examples": [
- "My SAST Scanner"
- ]
- },
- "url": {
- "type": "string",
- "description": "A link to more information about the scanner.",
- "examples": [
- "https://scanner.url"
- ]
- },
- "version": {
- "type": "string",
- "description": "The version of the scanner.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- },
- "vendor": {
- "type": "object",
- "description": "The vendor/maintainer of the scanner.",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- }
- }
- },
- "start_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-02-14T16:01:59"
- ]
- },
- "status": {
- "type": "string",
- "description": "Result of the scan.",
- "enum": [
- "success",
- "failure"
- ]
- },
- "type": {
- "type": "string",
- "description": "Type of the scan.",
- "enum": [
- "coverage_fuzzing"
- ]
- }
- }
- },
- "schema": {
- "type": "string",
- "description": "URI pointing to the validating security report schema.",
- "format": "uri"
- },
- "version": {
- "type": "string",
- "description": "The version of the schema to which the JSON report conforms.",
- "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
- },
- "vulnerabilities": {
- "type": "array",
- "description": "Array of vulnerability objects.",
- "items": {
- "type": "object",
- "description": "Describes the vulnerability.",
- "required": [
- "category",
- "cve",
- "identifiers",
- "location",
- "scanner"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
- "examples": [
- "642735a5-1425-428d-8d4e-3c854885a3c9"
- ]
- },
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
- "name": {
- "type": "string",
- "description": "The name of the vulnerability. This must not include the finding's specific information."
- },
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
- "description": {
- "type": "string",
- "description": "A long text section describing the vulnerability more fully."
- },
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
- "severity": {
- "type": "string",
- "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Info",
- "Unknown",
- "Low",
- "Medium",
- "High",
- "Critical"
- ]
- },
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
- "solution": {
- "type": "string",
- "description": "Explanation of how to fix the vulnerability."
- },
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
- "identifiers": {
- "type": "array",
- "minItems": 1,
- "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
- "items": {
- "type": "object",
- "required": [
- "type",
- "name",
- "value"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
- "minLength": 1
- },
- "name": {
- "type": "string",
- "description": "Human-readable name of the identifier.",
- "minLength": 1
- },
- "url": {
- "type": "string",
- "description": "URL of the identifier's documentation.",
- "format": "uri"
- },
- "value": {
- "type": "string",
- "description": "Value of the identifier, for matching purpose.",
- "minLength": 1
- }
- }
- }
- },
- "links": {
- "type": "array",
- "description": "An array of references to external documentation or articles that describe the vulnerability.",
- "items": {
- "type": "object",
- "required": [
- "url"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the vulnerability details link."
- },
- "url": {
- "type": "string",
- "description": "URL of the vulnerability details document.",
- "format": "uri"
- }
- }
- }
- },
- "details": {
- "$ref": "#/definitions/named_list/properties/items"
- },
- "location": {
- "description": "The location of the error",
- "type": "object",
- "properties": {
- "crash_address": {
- "type": "string",
- "description": "The relative address in memory were the crash occurred.",
- "examples": [
- "0xabababab"
- ]
- },
- "stacktrace_snippet": {
- "type": "string",
- "description": "The stack trace recorded during fuzzing resulting the crash.",
- "examples": [
- "func_a+0xabcd\nfunc_b+0xabcc"
- ]
- },
- "crash_state": {
- "type": "string",
- "description": "Minimised and normalized crash stack-trace (called crash_state).",
- "examples": [
- "func_a+0xa\nfunc_b+0xb\nfunc_c+0xc"
- ]
- },
- "crash_type": {
- "type": "string",
- "description": "Type of the crash.",
- "examples": [
- "Heap-Buffer-overflow",
- "Division-by-zero"
- ]
- }
- }
- }
- }
- }
- },
- "remediations": {
- "type": "array",
- "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
- "items": {
- "type": "object",
- "required": [
- "fixes",
- "summary",
- "diff"
- ],
- "properties": {
- "fixes": {
- "type": "array",
- "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
- "items": {
- "type": "object",
- "required": [
- "cve"
- ],
- "properties": {
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- }
- }
- }
- },
- "summary": {
- "type": "string",
- "minLength": 1,
- "description": "An overview of how the vulnerabilities were fixed."
- },
- "diff": {
- "type": "string",
- "minLength": 1,
- "description": "A base64-encoded remediation code diff, compatible with git apply."
- }
- }
- }
- }
- }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.0/dast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.0/dast-report-format.json
deleted file mode 100644
index 4d3868be019..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.0/dast-report-format.json
+++ /dev/null
@@ -1,1128 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Report format for GitLab DAST",
- "description": "This schema provides the the report format for Dynamic Application Security Testing (https://docs.gitlab.com/ee/user/application_security/dast).",
- "definitions": {
- "detail_type": {
- "oneOf": [
- {
- "$ref": "#/definitions/named_list"
- },
- {
- "$ref": "#/definitions/list"
- },
- {
- "$ref": "#/definitions/table"
- },
- {
- "$ref": "#/definitions/text"
- },
- {
- "$ref": "#/definitions/url"
- },
- {
- "$ref": "#/definitions/code"
- },
- {
- "$ref": "#/definitions/value"
- },
- {
- "$ref": "#/definitions/diff"
- },
- {
- "$ref": "#/definitions/markdown"
- },
- {
- "$ref": "#/definitions/commit"
- },
- {
- "$ref": "#/definitions/file_location"
- },
- {
- "$ref": "#/definitions/module_location"
- }
- ]
- },
- "text_value": {
- "type": "string"
- },
- "named_field": {
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "$ref": "#/definitions/text_value",
- "minLength": 1
- },
- "description": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "named_list": {
- "type": "object",
- "description": "An object with named and typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "named-list"
- },
- "items": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- {
- "$ref": "#/definitions/named_field"
- },
- {
- "$ref": "#/definitions/detail_type"
- }
- ]
- }
- }
- }
- }
- },
- "list": {
- "type": "object",
- "description": "A list of typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- },
- "table": {
- "type": "object",
- "description": "A table of typed fields",
- "required": [
- "type",
- "rows"
- ],
- "properties": {
- "type": {
- "const": "table"
- },
- "header": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- },
- "rows": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- }
- },
- "text": {
- "type": "object",
- "description": "Raw text",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "text"
- },
- "value": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "url": {
- "type": "object",
- "description": "A single URL",
- "required": [
- "type",
- "href"
- ],
- "properties": {
- "type": {
- "const": "url"
- },
- "text": {
- "$ref": "#/definitions/text_value"
- },
- "href": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "http://mysite.com"
- ]
- }
- }
- },
- "code": {
- "type": "object",
- "description": "A codeblock",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "code"
- },
- "value": {
- "type": "string"
- },
- "lang": {
- "type": "string",
- "description": "A programming language"
- }
- }
- },
- "value": {
- "type": "object",
- "description": "A field that can store a range of types of value",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "value"
- },
- "value": {
- "type": [
- "number",
- "string",
- "boolean"
- ]
- }
- }
- },
- "diff": {
- "type": "object",
- "description": "A diff",
- "required": [
- "type",
- "before",
- "after"
- ],
- "properties": {
- "type": {
- "const": "diff"
- },
- "before": {
- "type": "string"
- },
- "after": {
- "type": "string"
- }
- }
- },
- "markdown": {
- "type": "object",
- "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "markdown"
- },
- "value": {
- "$ref": "#/definitions/text_value",
- "examples": [
- "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
- ]
- }
- }
- },
- "commit": {
- "type": "object",
- "description": "A commit/tag/branch within the GitLab project",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "commit"
- },
- "value": {
- "type": "string",
- "description": "The commit SHA",
- "minLength": 1
- }
- }
- },
- "file_location": {
- "type": "object",
- "description": "A location within a file in the project",
- "required": [
- "type",
- "file_name",
- "line_start"
- ],
- "properties": {
- "type": {
- "const": "file-location"
- },
- "file_name": {
- "type": "string",
- "minLength": 1
- },
- "line_start": {
- "type": "integer"
- },
- "line_end": {
- "type": "integer"
- }
- }
- },
- "module_location": {
- "type": "object",
- "description": "A location within a binary module of the form module+relative_offset",
- "required": [
- "type",
- "module_name",
- "offset"
- ],
- "properties": {
- "type": {
- "const": "module-location"
- },
- "module_name": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "compiled_binary"
- ]
- },
- "offset": {
- "type": "integer",
- "examples": [
- 100
- ]
- }
- }
- }
- },
- "self": {
- "version": "14.0.0"
- },
- "required": [
- "version",
- "vulnerabilities"
- ],
- "additionalProperties": true,
- "properties": {
- "scan": {
- "type": "object",
- "required": [
- "end_time",
- "scanned_resources",
- "scanner",
- "start_time",
- "status",
- "type"
- ],
- "properties": {
- "end_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-01-28T03:26:02"
- ]
- },
- "messages": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Communication intended for the initiator of a scan.",
- "required": [
- "level",
- "value"
- ],
- "properties": {
- "level": {
- "type": "string",
- "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
- "enum": [
- "info",
- "warn",
- "fatal"
- ],
- "examples": [
- "info"
- ]
- },
- "value": {
- "type": "string",
- "description": "The message to communicate.",
- "minLength": 1,
- "examples": [
- "Permission denied, scanning aborted"
- ]
- }
- }
- }
- },
- "scanner": {
- "type": "object",
- "description": "Object defining the scanner used to perform the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the scanner.",
- "minLength": 1,
- "examples": [
- "my-sast-scanner"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the scanner, not required to be unique.",
- "minLength": 1,
- "examples": [
- "My SAST Scanner"
- ]
- },
- "url": {
- "type": "string",
- "description": "A link to more information about the scanner.",
- "examples": [
- "https://scanner.url"
- ]
- },
- "version": {
- "type": "string",
- "description": "The version of the scanner.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- },
- "vendor": {
- "type": "object",
- "description": "The vendor/maintainer of the scanner.",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- }
- }
- },
- "start_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-02-14T16:01:59"
- ]
- },
- "status": {
- "type": "string",
- "description": "Result of the scan.",
- "enum": [
- "success",
- "failure"
- ]
- },
- "type": {
- "type": "string",
- "description": "Type of the scan.",
- "enum": [
- "dast",
- "api_fuzzing"
- ]
- },
- "scanned_resources": {
- "type": "array",
- "description": "The attack surface scanned by DAST.",
- "items": {
- "type": "object",
- "required": [
- "method",
- "url",
- "type"
- ],
- "properties": {
- "method": {
- "type": "string",
- "minLength": 1,
- "description": "HTTP method of the scanned resource.",
- "examples": [
- "GET",
- "POST",
- "HEAD"
- ]
- },
- "url": {
- "type": "string",
- "minLength": 1,
- "description": "URL of the scanned resource.",
- "examples": [
- "http://my.site.com/a-page"
- ]
- },
- "type": {
- "type": "string",
- "minLength": 1,
- "description": "Type of the scanned resource, for DAST, this must be 'url'.",
- "examples": [
- "url"
- ]
- }
- }
- }
- }
- }
- },
- "schema": {
- "type": "string",
- "description": "URI pointing to the validating security report schema.",
- "format": "uri"
- },
- "version": {
- "type": "string",
- "description": "The version of the schema to which the JSON report conforms.",
- "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
- },
- "vulnerabilities": {
- "type": "array",
- "description": "Array of vulnerability objects.",
- "items": {
- "type": "object",
- "description": "Describes the vulnerability.",
- "required": [
- "category",
- "cve",
- "identifiers",
- "location",
- "scanner"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
- "examples": [
- "642735a5-1425-428d-8d4e-3c854885a3c9"
- ]
- },
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
- "name": {
- "type": "string",
- "description": "The name of the vulnerability. This must not include the finding's specific information."
- },
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
- "description": {
- "type": "string",
- "description": "A long text section describing the vulnerability more fully."
- },
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
- "severity": {
- "type": "string",
- "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Info",
- "Unknown",
- "Low",
- "Medium",
- "High",
- "Critical"
- ]
- },
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
- "solution": {
- "type": "string",
- "description": "Explanation of how to fix the vulnerability."
- },
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
- "identifiers": {
- "type": "array",
- "minItems": 1,
- "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
- "items": {
- "type": "object",
- "required": [
- "type",
- "name",
- "value"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
- "minLength": 1
- },
- "name": {
- "type": "string",
- "description": "Human-readable name of the identifier.",
- "minLength": 1
- },
- "url": {
- "type": "string",
- "description": "URL of the identifier's documentation.",
- "format": "uri"
- },
- "value": {
- "type": "string",
- "description": "Value of the identifier, for matching purpose.",
- "minLength": 1
- }
- }
- }
- },
- "links": {
- "type": "array",
- "description": "An array of references to external documentation or articles that describe the vulnerability.",
- "items": {
- "type": "object",
- "required": [
- "url"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the vulnerability details link."
- },
- "url": {
- "type": "string",
- "description": "URL of the vulnerability details document.",
- "format": "uri"
- }
- }
- }
- },
- "details": {
- "$ref": "#/definitions/named_list/properties/items"
- },
- "evidence": {
- "type": "object",
- "properties": {
- "source": {
- "type": "object",
- "description": "Source of evidence",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "Unique source identifier",
- "examples": [
- "assert:LogAnalysis",
- "assert:StatusCode"
- ]
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Source display name",
- "examples": [
- "Log Analysis",
- "Status Code"
- ]
- },
- "url": {
- "type": "string",
- "description": "Link to additional information",
- "examples": [
- "https://docs.gitlab.com/ee/development/integrations/secure.html"
- ]
- }
- }
- },
- "summary": {
- "type": "string",
- "description": "Human readable string containing evidence of the vulnerability.",
- "examples": [
- "Credit card 4111111111111111 found",
- "Server leaked information nginx/1.17.6"
- ]
- },
- "request": {
- "type": "object",
- "description": "An HTTP request.",
- "required": [
- "headers",
- "method",
- "url"
- ],
- "properties": {
- "headers": {
- "type": "array",
- "description": "HTTP headers present on the request.",
- "items": {
- "type": "object",
- "required": [
- "name",
- "value"
- ],
- "properties": {
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Name of the HTTP header.",
- "examples": [
- "Accept",
- "Content-Length",
- "Content-Type"
- ]
- },
- "value": {
- "type": "string",
- "minLength": 1,
- "description": "Value of the HTTP header.",
- "examples": [
- "*/*",
- "560",
- "application/json; charset=utf-8"
- ]
- }
- }
- }
- },
- "method": {
- "type": "string",
- "minLength": 1,
- "description": "HTTP method used in the request.",
- "examples": [
- "GET",
- "POST"
- ]
- },
- "url": {
- "type": "string",
- "minLength": 1,
- "description": "URL of the request.",
- "examples": [
- "http://my.site.com/vulnerable-endpoint?show-credit-card"
- ]
- },
- "body": {
- "type": "string",
- "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.",
- "examples": [
- "user=jsmith&first=%27&last=smith"
- ]
- }
- }
- },
- "response": {
- "type": "object",
- "description": "An HTTP response.",
- "required": [
- "headers",
- "reason_phrase",
- "status_code"
- ],
- "properties": {
- "headers": {
- "type": "array",
- "description": "HTTP headers present on the request.",
- "items": {
- "type": "object",
- "required": [
- "name",
- "value"
- ],
- "properties": {
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Name of the HTTP header.",
- "examples": [
- "Accept",
- "Content-Length",
- "Content-Type"
- ]
- },
- "value": {
- "type": "string",
- "minLength": 1,
- "description": "Value of the HTTP header.",
- "examples": [
- "*/*",
- "560",
- "application/json; charset=utf-8"
- ]
- }
- }
- }
- },
- "reason_phrase": {
- "type": "string",
- "description": "HTTP reason phrase of the response.",
- "examples": [
- "OK",
- "Internal Server Error"
- ]
- },
- "status_code": {
- "type": "integer",
- "description": "HTTP status code of the response.",
- "examples": [
- 200,
- 500
- ]
- },
- "body": {
- "type": "string",
- "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.",
- "examples": [
- "{\"user_id\": 2}"
- ]
- }
- }
- },
- "supporting_messages": {
- "type": "array",
- "description": "Array of supporting http messages.",
- "items": {
- "type": "object",
- "description": "A supporting http message.",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Message display name.",
- "examples": [
- "Unmodified",
- "Recorded"
- ]
- },
- "request": {
- "type": "object",
- "description": "An HTTP request.",
- "required": [
- "headers",
- "method",
- "url"
- ],
- "properties": {
- "headers": {
- "type": "array",
- "description": "HTTP headers present on the request.",
- "items": {
- "type": "object",
- "required": [
- "name",
- "value"
- ],
- "properties": {
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Name of the HTTP header.",
- "examples": [
- "Accept",
- "Content-Length",
- "Content-Type"
- ]
- },
- "value": {
- "type": "string",
- "minLength": 1,
- "description": "Value of the HTTP header.",
- "examples": [
- "*/*",
- "560",
- "application/json; charset=utf-8"
- ]
- }
- }
- }
- },
- "method": {
- "type": "string",
- "minLength": 1,
- "description": "HTTP method used in the request.",
- "examples": [
- "GET",
- "POST"
- ]
- },
- "url": {
- "type": "string",
- "minLength": 1,
- "description": "URL of the request.",
- "examples": [
- "http://my.site.com/vulnerable-endpoint?show-credit-card"
- ]
- },
- "body": {
- "type": "string",
- "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.",
- "examples": [
- "user=jsmith&first=%27&last=smith"
- ]
- }
- }
- },
- "response": {
- "type": "object",
- "description": "An HTTP response.",
- "required": [
- "headers",
- "reason_phrase",
- "status_code"
- ],
- "properties": {
- "headers": {
- "type": "array",
- "description": "HTTP headers present on the request.",
- "items": {
- "type": "object",
- "required": [
- "name",
- "value"
- ],
- "properties": {
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Name of the HTTP header.",
- "examples": [
- "Accept",
- "Content-Length",
- "Content-Type"
- ]
- },
- "value": {
- "type": "string",
- "minLength": 1,
- "description": "Value of the HTTP header.",
- "examples": [
- "*/*",
- "560",
- "application/json; charset=utf-8"
- ]
- }
- }
- }
- },
- "reason_phrase": {
- "type": "string",
- "description": "HTTP reason phrase of the response.",
- "examples": [
- "OK",
- "Internal Server Error"
- ]
- },
- "status_code": {
- "type": "integer",
- "description": "HTTP status code of the response.",
- "examples": [
- 200,
- 500
- ]
- },
- "body": {
- "type": "string",
- "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.",
- "examples": [
- "{\"user_id\": 2}"
- ]
- }
- }
- }
- }
- }
- }
- }
- },
- "location": {
- "type": "object",
- "description": "Identifies the vulnerability's location.",
- "properties": {
- "hostname": {
- "type": "string",
- "description": "The protocol, domain, and port of the application where the vulnerability was found."
- },
- "method": {
- "type": "string",
- "description": "The HTTP method that was used to request the URL where the vulnerability was found."
- },
- "param": {
- "type": "string",
- "description": "A value provided by a vulnerability rule related to the found vulnerability. Examples include a header value, or a parameter used in a HTTP POST."
- },
- "path": {
- "type": "string",
- "description": "The path of the URL where the vulnerability was found. Typically, this would start with a forward slash."
- }
- }
- },
- "assets": {
- "type": "array",
- "description": "Array of build assets associated with vulnerability.",
- "items": {
- "type": "object",
- "description": "Describes an asset associated with vulnerability.",
- "required": [
- "type",
- "name",
- "url"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "The type of asset",
- "enum": [
- "http_session",
- "postman"
- ]
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Display name for asset",
- "examples": [
- "HTTP Messages",
- "Postman Collection"
- ]
- },
- "url": {
- "type": "string",
- "minLength": 1,
- "description": "Link to asset in build artifacts",
- "examples": [
- "https://gitlab.com/gitlab-org/security-products/dast/-/jobs/626397001/artifacts/file//output/zap_session.data"
- ]
- }
- }
- }
- },
- "discovered_at": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss.sss, representing when the vulnerability was discovered",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}\\.\\d{3}$",
- "examples": [
- "2020-01-28T03:26:02.956"
- ]
- }
- }
- }
- },
- "remediations": {
- "type": "array",
- "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
- "items": {
- "type": "object",
- "required": [
- "fixes",
- "summary",
- "diff"
- ],
- "properties": {
- "fixes": {
- "type": "array",
- "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
- "items": {
- "type": "object",
- "required": [
- "cve"
- ],
- "properties": {
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- }
- }
- }
- },
- "summary": {
- "type": "string",
- "minLength": 1,
- "description": "An overview of how the vulnerabilities were fixed."
- },
- "diff": {
- "type": "string",
- "minLength": 1,
- "description": "A base64-encoded remediation code diff, compatible with git apply."
- }
- }
- }
- }
- }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.0/dependency-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.0/dependency-scanning-report-format.json
deleted file mode 100644
index f0c1a90adcc..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.0/dependency-scanning-report-format.json
+++ /dev/null
@@ -1,805 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Report format for GitLab Dependency Scanning",
- "description": "This schema provides the the report format for Dependency Scanning analyzers (https://docs.gitlab.com/ee/user/application_security/dependency_scanning).",
- "definitions": {
- "detail_type": {
- "oneOf": [
- {
- "$ref": "#/definitions/named_list"
- },
- {
- "$ref": "#/definitions/list"
- },
- {
- "$ref": "#/definitions/table"
- },
- {
- "$ref": "#/definitions/text"
- },
- {
- "$ref": "#/definitions/url"
- },
- {
- "$ref": "#/definitions/code"
- },
- {
- "$ref": "#/definitions/value"
- },
- {
- "$ref": "#/definitions/diff"
- },
- {
- "$ref": "#/definitions/markdown"
- },
- {
- "$ref": "#/definitions/commit"
- },
- {
- "$ref": "#/definitions/file_location"
- },
- {
- "$ref": "#/definitions/module_location"
- }
- ]
- },
- "text_value": {
- "type": "string"
- },
- "named_field": {
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "$ref": "#/definitions/text_value",
- "minLength": 1
- },
- "description": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "named_list": {
- "type": "object",
- "description": "An object with named and typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "named-list"
- },
- "items": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- {
- "$ref": "#/definitions/named_field"
- },
- {
- "$ref": "#/definitions/detail_type"
- }
- ]
- }
- }
- }
- }
- },
- "list": {
- "type": "object",
- "description": "A list of typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- },
- "table": {
- "type": "object",
- "description": "A table of typed fields",
- "required": [
- "type",
- "rows"
- ],
- "properties": {
- "type": {
- "const": "table"
- },
- "header": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- },
- "rows": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- }
- },
- "text": {
- "type": "object",
- "description": "Raw text",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "text"
- },
- "value": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "url": {
- "type": "object",
- "description": "A single URL",
- "required": [
- "type",
- "href"
- ],
- "properties": {
- "type": {
- "const": "url"
- },
- "text": {
- "$ref": "#/definitions/text_value"
- },
- "href": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "http://mysite.com"
- ]
- }
- }
- },
- "code": {
- "type": "object",
- "description": "A codeblock",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "code"
- },
- "value": {
- "type": "string"
- },
- "lang": {
- "type": "string",
- "description": "A programming language"
- }
- }
- },
- "value": {
- "type": "object",
- "description": "A field that can store a range of types of value",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "value"
- },
- "value": {
- "type": [
- "number",
- "string",
- "boolean"
- ]
- }
- }
- },
- "diff": {
- "type": "object",
- "description": "A diff",
- "required": [
- "type",
- "before",
- "after"
- ],
- "properties": {
- "type": {
- "const": "diff"
- },
- "before": {
- "type": "string"
- },
- "after": {
- "type": "string"
- }
- }
- },
- "markdown": {
- "type": "object",
- "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "markdown"
- },
- "value": {
- "$ref": "#/definitions/text_value",
- "examples": [
- "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
- ]
- }
- }
- },
- "commit": {
- "type": "object",
- "description": "A commit/tag/branch within the GitLab project",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "commit"
- },
- "value": {
- "type": "string",
- "description": "The commit SHA",
- "minLength": 1
- }
- }
- },
- "file_location": {
- "type": "object",
- "description": "A location within a file in the project",
- "required": [
- "type",
- "file_name",
- "line_start"
- ],
- "properties": {
- "type": {
- "const": "file-location"
- },
- "file_name": {
- "type": "string",
- "minLength": 1
- },
- "line_start": {
- "type": "integer"
- },
- "line_end": {
- "type": "integer"
- }
- }
- },
- "module_location": {
- "type": "object",
- "description": "A location within a binary module of the form module+relative_offset",
- "required": [
- "type",
- "module_name",
- "offset"
- ],
- "properties": {
- "type": {
- "const": "module-location"
- },
- "module_name": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "compiled_binary"
- ]
- },
- "offset": {
- "type": "integer",
- "examples": [
- 100
- ]
- }
- }
- }
- },
- "self": {
- "version": "14.0.0"
- },
- "required": [
- "dependency_files",
- "version",
- "vulnerabilities"
- ],
- "additionalProperties": true,
- "properties": {
- "scan": {
- "type": "object",
- "required": [
- "end_time",
- "scanner",
- "start_time",
- "status",
- "type"
- ],
- "properties": {
- "end_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-01-28T03:26:02"
- ]
- },
- "messages": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Communication intended for the initiator of a scan.",
- "required": [
- "level",
- "value"
- ],
- "properties": {
- "level": {
- "type": "string",
- "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
- "enum": [
- "info",
- "warn",
- "fatal"
- ],
- "examples": [
- "info"
- ]
- },
- "value": {
- "type": "string",
- "description": "The message to communicate.",
- "minLength": 1,
- "examples": [
- "Permission denied, scanning aborted"
- ]
- }
- }
- }
- },
- "scanner": {
- "type": "object",
- "description": "Object defining the scanner used to perform the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the scanner.",
- "minLength": 1,
- "examples": [
- "my-sast-scanner"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the scanner, not required to be unique.",
- "minLength": 1,
- "examples": [
- "My SAST Scanner"
- ]
- },
- "url": {
- "type": "string",
- "description": "A link to more information about the scanner.",
- "examples": [
- "https://scanner.url"
- ]
- },
- "version": {
- "type": "string",
- "description": "The version of the scanner.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- },
- "vendor": {
- "type": "object",
- "description": "The vendor/maintainer of the scanner.",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- }
- }
- },
- "start_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-02-14T16:01:59"
- ]
- },
- "status": {
- "type": "string",
- "description": "Result of the scan.",
- "enum": [
- "success",
- "failure"
- ]
- },
- "type": {
- "type": "string",
- "description": "Type of the scan.",
- "enum": [
- "dependency_scanning"
- ]
- }
- }
- },
- "schema": {
- "type": "string",
- "description": "URI pointing to the validating security report schema.",
- "format": "uri"
- },
- "version": {
- "type": "string",
- "description": "The version of the schema to which the JSON report conforms.",
- "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
- },
- "vulnerabilities": {
- "type": "array",
- "description": "Array of vulnerability objects.",
- "items": {
- "type": "object",
- "description": "Describes the vulnerability.",
- "required": [
- "category",
- "cve",
- "identifiers",
- "location",
- "scanner"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
- "examples": [
- "642735a5-1425-428d-8d4e-3c854885a3c9"
- ]
- },
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
- "name": {
- "type": "string",
- "description": "The name of the vulnerability. This must not include the finding's specific information."
- },
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
- "description": {
- "type": "string",
- "description": "A long text section describing the vulnerability more fully."
- },
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
- "severity": {
- "type": "string",
- "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Info",
- "Unknown",
- "Low",
- "Medium",
- "High",
- "Critical"
- ]
- },
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
- "solution": {
- "type": "string",
- "description": "Explanation of how to fix the vulnerability."
- },
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
- "identifiers": {
- "type": "array",
- "minItems": 1,
- "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
- "items": {
- "type": "object",
- "required": [
- "type",
- "name",
- "value"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
- "minLength": 1
- },
- "name": {
- "type": "string",
- "description": "Human-readable name of the identifier.",
- "minLength": 1
- },
- "url": {
- "type": "string",
- "description": "URL of the identifier's documentation.",
- "format": "uri"
- },
- "value": {
- "type": "string",
- "description": "Value of the identifier, for matching purpose.",
- "minLength": 1
- }
- }
- }
- },
- "links": {
- "type": "array",
- "description": "An array of references to external documentation or articles that describe the vulnerability.",
- "items": {
- "type": "object",
- "required": [
- "url"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the vulnerability details link."
- },
- "url": {
- "type": "string",
- "description": "URL of the vulnerability details document.",
- "format": "uri"
- }
- }
- }
- },
- "details": {
- "$ref": "#/definitions/named_list/properties/items"
- },
- "location": {
- "type": "object",
- "description": "Identifies the vulnerability's location.",
- "required": [
- "file",
- "dependency"
- ],
- "properties": {
- "file": {
- "type": "string",
- "minLength": 1,
- "description": "Path to the manifest or lock file where the dependency is declared (such as yarn.lock)."
- },
- "dependency": {
- "type": "object",
- "description": "Describes the dependency of a project where the vulnerability is located.",
- "properties": {
- "package": {
- "type": "object",
- "description": "Provides information on the package where the vulnerability is located.",
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the package where the vulnerability is located."
- }
- }
- },
- "version": {
- "type": "string",
- "description": "Version of the vulnerable package."
- },
- "iid": {
- "description": "ID that identifies the dependency in the scope of a dependency file.",
- "type": "number"
- },
- "direct": {
- "type": "boolean",
- "description": "Tells whether this is a direct, top-level dependency of the scanned project."
- },
- "dependency_path": {
- "type": "array",
- "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.",
- "items": {
- "type": "object",
- "required": [
- "iid"
- ],
- "properties": {
- "iid": {
- "type": "number",
- "description": "ID that is unique in the scope of a parent object, and specific to the resource type."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- }
- },
- "remediations": {
- "type": "array",
- "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
- "items": {
- "type": "object",
- "required": [
- "fixes",
- "summary",
- "diff"
- ],
- "properties": {
- "fixes": {
- "type": "array",
- "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
- "items": {
- "type": "object",
- "required": [
- "cve"
- ],
- "properties": {
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- }
- }
- }
- },
- "summary": {
- "type": "string",
- "minLength": 1,
- "description": "An overview of how the vulnerabilities were fixed."
- },
- "diff": {
- "type": "string",
- "minLength": 1,
- "description": "A base64-encoded remediation code diff, compatible with git apply."
- }
- }
- }
- },
- "dependency_files": {
- "type": "array",
- "description": "List of dependency files identified in the project.",
- "items": {
- "type": "object",
- "required": [
- "path",
- "package_manager",
- "dependencies"
- ],
- "properties": {
- "path": {
- "type": "string",
- "minLength": 1
- },
- "package_manager": {
- "type": "string",
- "minLength": 1
- },
- "dependencies": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Describes the dependency of a project where the vulnerability is located.",
- "properties": {
- "package": {
- "type": "object",
- "description": "Provides information on the package where the vulnerability is located.",
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the package where the vulnerability is located."
- }
- }
- },
- "version": {
- "type": "string",
- "description": "Version of the vulnerable package."
- },
- "iid": {
- "description": "ID that identifies the dependency in the scope of a dependency file.",
- "type": "number"
- },
- "direct": {
- "type": "boolean",
- "description": "Tells whether this is a direct, top-level dependency of the scanned project."
- },
- "dependency_path": {
- "type": "array",
- "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.",
- "items": {
- "type": "object",
- "required": [
- "iid"
- ],
- "properties": {
- "iid": {
- "type": "number",
- "description": "ID that is unique in the scope of a parent object, and specific to the resource type."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- }
- }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.0/sast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.0/sast-report-format.json
deleted file mode 100644
index a7159be0190..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.0/sast-report-format.json
+++ /dev/null
@@ -1,706 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Report format for GitLab SAST",
- "description": "This schema provides the report format for Static Application Security Testing analyzers (https://docs.gitlab.com/ee/user/application_security/sast).",
- "definitions": {
- "detail_type": {
- "oneOf": [
- {
- "$ref": "#/definitions/named_list"
- },
- {
- "$ref": "#/definitions/list"
- },
- {
- "$ref": "#/definitions/table"
- },
- {
- "$ref": "#/definitions/text"
- },
- {
- "$ref": "#/definitions/url"
- },
- {
- "$ref": "#/definitions/code"
- },
- {
- "$ref": "#/definitions/value"
- },
- {
- "$ref": "#/definitions/diff"
- },
- {
- "$ref": "#/definitions/markdown"
- },
- {
- "$ref": "#/definitions/commit"
- },
- {
- "$ref": "#/definitions/file_location"
- },
- {
- "$ref": "#/definitions/module_location"
- }
- ]
- },
- "text_value": {
- "type": "string"
- },
- "named_field": {
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "$ref": "#/definitions/text_value",
- "minLength": 1
- },
- "description": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "named_list": {
- "type": "object",
- "description": "An object with named and typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "named-list"
- },
- "items": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- {
- "$ref": "#/definitions/named_field"
- },
- {
- "$ref": "#/definitions/detail_type"
- }
- ]
- }
- }
- }
- }
- },
- "list": {
- "type": "object",
- "description": "A list of typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- },
- "table": {
- "type": "object",
- "description": "A table of typed fields",
- "required": [
- "type",
- "rows"
- ],
- "properties": {
- "type": {
- "const": "table"
- },
- "header": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- },
- "rows": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- }
- },
- "text": {
- "type": "object",
- "description": "Raw text",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "text"
- },
- "value": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "url": {
- "type": "object",
- "description": "A single URL",
- "required": [
- "type",
- "href"
- ],
- "properties": {
- "type": {
- "const": "url"
- },
- "text": {
- "$ref": "#/definitions/text_value"
- },
- "href": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "http://mysite.com"
- ]
- }
- }
- },
- "code": {
- "type": "object",
- "description": "A codeblock",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "code"
- },
- "value": {
- "type": "string"
- },
- "lang": {
- "type": "string",
- "description": "A programming language"
- }
- }
- },
- "value": {
- "type": "object",
- "description": "A field that can store a range of types of value",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "value"
- },
- "value": {
- "type": [
- "number",
- "string",
- "boolean"
- ]
- }
- }
- },
- "diff": {
- "type": "object",
- "description": "A diff",
- "required": [
- "type",
- "before",
- "after"
- ],
- "properties": {
- "type": {
- "const": "diff"
- },
- "before": {
- "type": "string"
- },
- "after": {
- "type": "string"
- }
- }
- },
- "markdown": {
- "type": "object",
- "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "markdown"
- },
- "value": {
- "$ref": "#/definitions/text_value",
- "examples": [
- "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
- ]
- }
- }
- },
- "commit": {
- "type": "object",
- "description": "A commit/tag/branch within the GitLab project",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "commit"
- },
- "value": {
- "type": "string",
- "description": "The commit SHA",
- "minLength": 1
- }
- }
- },
- "file_location": {
- "type": "object",
- "description": "A location within a file in the project",
- "required": [
- "type",
- "file_name",
- "line_start"
- ],
- "properties": {
- "type": {
- "const": "file-location"
- },
- "file_name": {
- "type": "string",
- "minLength": 1
- },
- "line_start": {
- "type": "integer"
- },
- "line_end": {
- "type": "integer"
- }
- }
- },
- "module_location": {
- "type": "object",
- "description": "A location within a binary module of the form module+relative_offset",
- "required": [
- "type",
- "module_name",
- "offset"
- ],
- "properties": {
- "type": {
- "const": "module-location"
- },
- "module_name": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "compiled_binary"
- ]
- },
- "offset": {
- "type": "integer",
- "examples": [
- 100
- ]
- }
- }
- }
- },
- "self": {
- "version": "14.0.0"
- },
- "required": [
- "version",
- "vulnerabilities"
- ],
- "additionalProperties": true,
- "properties": {
- "scan": {
- "type": "object",
- "required": [
- "end_time",
- "scanner",
- "start_time",
- "status",
- "type"
- ],
- "properties": {
- "end_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-01-28T03:26:02"
- ]
- },
- "messages": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Communication intended for the initiator of a scan.",
- "required": [
- "level",
- "value"
- ],
- "properties": {
- "level": {
- "type": "string",
- "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
- "enum": [
- "info",
- "warn",
- "fatal"
- ],
- "examples": [
- "info"
- ]
- },
- "value": {
- "type": "string",
- "description": "The message to communicate.",
- "minLength": 1,
- "examples": [
- "Permission denied, scanning aborted"
- ]
- }
- }
- }
- },
- "scanner": {
- "type": "object",
- "description": "Object defining the scanner used to perform the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the scanner.",
- "minLength": 1,
- "examples": [
- "my-sast-scanner"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the scanner, not required to be unique.",
- "minLength": 1,
- "examples": [
- "My SAST Scanner"
- ]
- },
- "url": {
- "type": "string",
- "description": "A link to more information about the scanner.",
- "examples": [
- "https://scanner.url"
- ]
- },
- "version": {
- "type": "string",
- "description": "The version of the scanner.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- },
- "vendor": {
- "type": "object",
- "description": "The vendor/maintainer of the scanner.",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- }
- }
- },
- "start_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-02-14T16:01:59"
- ]
- },
- "status": {
- "type": "string",
- "description": "Result of the scan.",
- "enum": [
- "success",
- "failure"
- ]
- },
- "type": {
- "type": "string",
- "description": "Type of the scan.",
- "enum": [
- "sast"
- ]
- }
- }
- },
- "schema": {
- "type": "string",
- "description": "URI pointing to the validating security report schema.",
- "format": "uri"
- },
- "version": {
- "type": "string",
- "description": "The version of the schema to which the JSON report conforms.",
- "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
- },
- "vulnerabilities": {
- "type": "array",
- "description": "Array of vulnerability objects.",
- "items": {
- "type": "object",
- "description": "Describes the vulnerability.",
- "required": [
- "category",
- "cve",
- "identifiers",
- "location",
- "scanner"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
- "examples": [
- "642735a5-1425-428d-8d4e-3c854885a3c9"
- ]
- },
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
- "name": {
- "type": "string",
- "description": "The name of the vulnerability. This must not include the finding's specific information."
- },
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
- "description": {
- "type": "string",
- "description": "A long text section describing the vulnerability more fully."
- },
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
- "severity": {
- "type": "string",
- "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Info",
- "Unknown",
- "Low",
- "Medium",
- "High",
- "Critical"
- ]
- },
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
- "solution": {
- "type": "string",
- "description": "Explanation of how to fix the vulnerability."
- },
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
- "identifiers": {
- "type": "array",
- "minItems": 1,
- "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
- "items": {
- "type": "object",
- "required": [
- "type",
- "name",
- "value"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
- "minLength": 1
- },
- "name": {
- "type": "string",
- "description": "Human-readable name of the identifier.",
- "minLength": 1
- },
- "url": {
- "type": "string",
- "description": "URL of the identifier's documentation.",
- "format": "uri"
- },
- "value": {
- "type": "string",
- "description": "Value of the identifier, for matching purpose.",
- "minLength": 1
- }
- }
- }
- },
- "links": {
- "type": "array",
- "description": "An array of references to external documentation or articles that describe the vulnerability.",
- "items": {
- "type": "object",
- "required": [
- "url"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the vulnerability details link."
- },
- "url": {
- "type": "string",
- "description": "URL of the vulnerability details document.",
- "format": "uri"
- }
- }
- }
- },
- "details": {
- "$ref": "#/definitions/named_list/properties/items"
- },
- "location": {
- "type": "object",
- "description": "Identifies the vulnerability's location.",
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the code affected by the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the code affected by the vulnerability."
- },
- "class": {
- "type": "string",
- "description": "Provides the name of the class where the vulnerability is located."
- },
- "method": {
- "type": "string",
- "description": "Provides the name of the method where the vulnerability is located."
- }
- }
- },
- "raw_source_code_extract": {
- "type": "string",
- "description": "Provides an unsanitized excerpt of the affected source code."
- }
- }
- }
- },
- "remediations": {
- "type": "array",
- "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
- "items": {
- "type": "object",
- "required": [
- "fixes",
- "summary",
- "diff"
- ],
- "properties": {
- "fixes": {
- "type": "array",
- "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
- "items": {
- "type": "object",
- "required": [
- "cve"
- ],
- "properties": {
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- }
- }
- }
- },
- "summary": {
- "type": "string",
- "minLength": 1,
- "description": "An overview of how the vulnerabilities were fixed."
- },
- "diff": {
- "type": "string",
- "minLength": 1,
- "description": "A base64-encoded remediation code diff, compatible with git apply."
- }
- }
- }
- }
- }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.0/secret-detection-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.0/secret-detection-report-format.json
deleted file mode 100644
index 462e23a151c..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.0/secret-detection-report-format.json
+++ /dev/null
@@ -1,729 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Report format for GitLab Secret Detection",
- "description": "This schema provides the the report format for the Secret Detection analyzer (https://docs.gitlab.com/ee/user/application_security/secret_detection)",
- "definitions": {
- "detail_type": {
- "oneOf": [
- {
- "$ref": "#/definitions/named_list"
- },
- {
- "$ref": "#/definitions/list"
- },
- {
- "$ref": "#/definitions/table"
- },
- {
- "$ref": "#/definitions/text"
- },
- {
- "$ref": "#/definitions/url"
- },
- {
- "$ref": "#/definitions/code"
- },
- {
- "$ref": "#/definitions/value"
- },
- {
- "$ref": "#/definitions/diff"
- },
- {
- "$ref": "#/definitions/markdown"
- },
- {
- "$ref": "#/definitions/commit"
- },
- {
- "$ref": "#/definitions/file_location"
- },
- {
- "$ref": "#/definitions/module_location"
- }
- ]
- },
- "text_value": {
- "type": "string"
- },
- "named_field": {
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "$ref": "#/definitions/text_value",
- "minLength": 1
- },
- "description": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "named_list": {
- "type": "object",
- "description": "An object with named and typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "named-list"
- },
- "items": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- {
- "$ref": "#/definitions/named_field"
- },
- {
- "$ref": "#/definitions/detail_type"
- }
- ]
- }
- }
- }
- }
- },
- "list": {
- "type": "object",
- "description": "A list of typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- },
- "table": {
- "type": "object",
- "description": "A table of typed fields",
- "required": [
- "type",
- "rows"
- ],
- "properties": {
- "type": {
- "const": "table"
- },
- "header": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- },
- "rows": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- }
- },
- "text": {
- "type": "object",
- "description": "Raw text",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "text"
- },
- "value": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "url": {
- "type": "object",
- "description": "A single URL",
- "required": [
- "type",
- "href"
- ],
- "properties": {
- "type": {
- "const": "url"
- },
- "text": {
- "$ref": "#/definitions/text_value"
- },
- "href": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "http://mysite.com"
- ]
- }
- }
- },
- "code": {
- "type": "object",
- "description": "A codeblock",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "code"
- },
- "value": {
- "type": "string"
- },
- "lang": {
- "type": "string",
- "description": "A programming language"
- }
- }
- },
- "value": {
- "type": "object",
- "description": "A field that can store a range of types of value",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "value"
- },
- "value": {
- "type": [
- "number",
- "string",
- "boolean"
- ]
- }
- }
- },
- "diff": {
- "type": "object",
- "description": "A diff",
- "required": [
- "type",
- "before",
- "after"
- ],
- "properties": {
- "type": {
- "const": "diff"
- },
- "before": {
- "type": "string"
- },
- "after": {
- "type": "string"
- }
- }
- },
- "markdown": {
- "type": "object",
- "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "markdown"
- },
- "value": {
- "$ref": "#/definitions/text_value",
- "examples": [
- "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
- ]
- }
- }
- },
- "commit": {
- "type": "object",
- "description": "A commit/tag/branch within the GitLab project",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "commit"
- },
- "value": {
- "type": "string",
- "description": "The commit SHA",
- "minLength": 1
- }
- }
- },
- "file_location": {
- "type": "object",
- "description": "A location within a file in the project",
- "required": [
- "type",
- "file_name",
- "line_start"
- ],
- "properties": {
- "type": {
- "const": "file-location"
- },
- "file_name": {
- "type": "string",
- "minLength": 1
- },
- "line_start": {
- "type": "integer"
- },
- "line_end": {
- "type": "integer"
- }
- }
- },
- "module_location": {
- "type": "object",
- "description": "A location within a binary module of the form module+relative_offset",
- "required": [
- "type",
- "module_name",
- "offset"
- ],
- "properties": {
- "type": {
- "const": "module-location"
- },
- "module_name": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "compiled_binary"
- ]
- },
- "offset": {
- "type": "integer",
- "examples": [
- 100
- ]
- }
- }
- }
- },
- "self": {
- "version": "14.0.0"
- },
- "required": [
- "version",
- "vulnerabilities"
- ],
- "additionalProperties": true,
- "properties": {
- "scan": {
- "type": "object",
- "required": [
- "end_time",
- "scanner",
- "start_time",
- "status",
- "type"
- ],
- "properties": {
- "end_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-01-28T03:26:02"
- ]
- },
- "messages": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Communication intended for the initiator of a scan.",
- "required": [
- "level",
- "value"
- ],
- "properties": {
- "level": {
- "type": "string",
- "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
- "enum": [
- "info",
- "warn",
- "fatal"
- ],
- "examples": [
- "info"
- ]
- },
- "value": {
- "type": "string",
- "description": "The message to communicate.",
- "minLength": 1,
- "examples": [
- "Permission denied, scanning aborted"
- ]
- }
- }
- }
- },
- "scanner": {
- "type": "object",
- "description": "Object defining the scanner used to perform the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the scanner.",
- "minLength": 1,
- "examples": [
- "my-sast-scanner"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the scanner, not required to be unique.",
- "minLength": 1,
- "examples": [
- "My SAST Scanner"
- ]
- },
- "url": {
- "type": "string",
- "description": "A link to more information about the scanner.",
- "examples": [
- "https://scanner.url"
- ]
- },
- "version": {
- "type": "string",
- "description": "The version of the scanner.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- },
- "vendor": {
- "type": "object",
- "description": "The vendor/maintainer of the scanner.",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- }
- }
- },
- "start_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-02-14T16:01:59"
- ]
- },
- "status": {
- "type": "string",
- "description": "Result of the scan.",
- "enum": [
- "success",
- "failure"
- ]
- },
- "type": {
- "type": "string",
- "description": "Type of the scan.",
- "enum": [
- "secret_detection"
- ]
- }
- }
- },
- "schema": {
- "type": "string",
- "description": "URI pointing to the validating security report schema.",
- "format": "uri"
- },
- "version": {
- "type": "string",
- "description": "The version of the schema to which the JSON report conforms.",
- "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
- },
- "vulnerabilities": {
- "type": "array",
- "description": "Array of vulnerability objects.",
- "items": {
- "type": "object",
- "description": "Describes the vulnerability.",
- "required": [
- "category",
- "cve",
- "identifiers",
- "location",
- "scanner"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
- "examples": [
- "642735a5-1425-428d-8d4e-3c854885a3c9"
- ]
- },
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
- "name": {
- "type": "string",
- "description": "The name of the vulnerability. This must not include the finding's specific information."
- },
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
- "description": {
- "type": "string",
- "description": "A long text section describing the vulnerability more fully."
- },
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
- "severity": {
- "type": "string",
- "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Info",
- "Unknown",
- "Low",
- "Medium",
- "High",
- "Critical"
- ]
- },
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
- "solution": {
- "type": "string",
- "description": "Explanation of how to fix the vulnerability."
- },
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
- "identifiers": {
- "type": "array",
- "minItems": 1,
- "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
- "items": {
- "type": "object",
- "required": [
- "type",
- "name",
- "value"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
- "minLength": 1
- },
- "name": {
- "type": "string",
- "description": "Human-readable name of the identifier.",
- "minLength": 1
- },
- "url": {
- "type": "string",
- "description": "URL of the identifier's documentation.",
- "format": "uri"
- },
- "value": {
- "type": "string",
- "description": "Value of the identifier, for matching purpose.",
- "minLength": 1
- }
- }
- }
- },
- "links": {
- "type": "array",
- "description": "An array of references to external documentation or articles that describe the vulnerability.",
- "items": {
- "type": "object",
- "required": [
- "url"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the vulnerability details link."
- },
- "url": {
- "type": "string",
- "description": "URL of the vulnerability details document.",
- "format": "uri"
- }
- }
- }
- },
- "details": {
- "$ref": "#/definitions/named_list/properties/items"
- },
- "location": {
- "required": [
- "commit"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located"
- },
- "commit": {
- "type": "object",
- "description": "Represents the commit in which the vulnerability was detected",
- "required": [
- "sha"
- ],
- "properties": {
- "author": {
- "type": "string"
- },
- "date": {
- "type": "string"
- },
- "message": {
- "type": "string"
- },
- "sha": {
- "type": "string",
- "minLength": 1
- }
- }
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the code affected by the vulnerability"
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the code affected by the vulnerability"
- },
- "class": {
- "type": "string",
- "description": "Provides the name of the class where the vulnerability is located"
- },
- "method": {
- "type": "string",
- "description": "Provides the name of the method where the vulnerability is located"
- }
- }
- },
- "raw_source_code_extract": {
- "type": "string",
- "description": "Provides an unsanitized excerpt of the affected source code."
- }
- }
- }
- },
- "remediations": {
- "type": "array",
- "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
- "items": {
- "type": "object",
- "required": [
- "fixes",
- "summary",
- "diff"
- ],
- "properties": {
- "fixes": {
- "type": "array",
- "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
- "items": {
- "type": "object",
- "required": [
- "cve"
- ],
- "properties": {
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- }
- }
- }
- },
- "summary": {
- "type": "string",
- "minLength": 1,
- "description": "An overview of how the vulnerabilities were fixed."
- },
- "diff": {
- "type": "string",
- "minLength": 1,
- "description": "A base64-encoded remediation code diff, compatible with git apply."
- }
- }
- }
- }
- }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.1/container-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.1/container-scanning-report-format.json
deleted file mode 100644
index d01e7818866..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.1/container-scanning-report-format.json
+++ /dev/null
@@ -1,809 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Report format for GitLab Container Scanning",
- "description": "This schema provides the the report format for Container Scanning (https://docs.gitlab.com/ee/user/application_security/container_scanning).",
- "definitions": {
- "detail_type": {
- "oneOf": [
- {
- "$ref": "#/definitions/named_list"
- },
- {
- "$ref": "#/definitions/list"
- },
- {
- "$ref": "#/definitions/table"
- },
- {
- "$ref": "#/definitions/text"
- },
- {
- "$ref": "#/definitions/url"
- },
- {
- "$ref": "#/definitions/code"
- },
- {
- "$ref": "#/definitions/value"
- },
- {
- "$ref": "#/definitions/diff"
- },
- {
- "$ref": "#/definitions/markdown"
- },
- {
- "$ref": "#/definitions/commit"
- },
- {
- "$ref": "#/definitions/file_location"
- },
- {
- "$ref": "#/definitions/module_location"
- }
- ]
- },
- "text_value": {
- "type": "string"
- },
- "named_field": {
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "$ref": "#/definitions/text_value",
- "minLength": 1
- },
- "description": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "named_list": {
- "type": "object",
- "description": "An object with named and typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "named-list"
- },
- "items": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- {
- "$ref": "#/definitions/named_field"
- },
- {
- "$ref": "#/definitions/detail_type"
- }
- ]
- }
- }
- }
- }
- },
- "list": {
- "type": "object",
- "description": "A list of typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- },
- "table": {
- "type": "object",
- "description": "A table of typed fields",
- "required": [
- "type",
- "rows"
- ],
- "properties": {
- "type": {
- "const": "table"
- },
- "header": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- },
- "rows": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- }
- },
- "text": {
- "type": "object",
- "description": "Raw text",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "text"
- },
- "value": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "url": {
- "type": "object",
- "description": "A single URL",
- "required": [
- "type",
- "href"
- ],
- "properties": {
- "type": {
- "const": "url"
- },
- "text": {
- "$ref": "#/definitions/text_value"
- },
- "href": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "http://mysite.com"
- ]
- }
- }
- },
- "code": {
- "type": "object",
- "description": "A codeblock",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "code"
- },
- "value": {
- "type": "string"
- },
- "lang": {
- "type": "string",
- "description": "A programming language"
- }
- }
- },
- "value": {
- "type": "object",
- "description": "A field that can store a range of types of value",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "value"
- },
- "value": {
- "type": [
- "number",
- "string",
- "boolean"
- ]
- }
- }
- },
- "diff": {
- "type": "object",
- "description": "A diff",
- "required": [
- "type",
- "before",
- "after"
- ],
- "properties": {
- "type": {
- "const": "diff"
- },
- "before": {
- "type": "string"
- },
- "after": {
- "type": "string"
- }
- }
- },
- "markdown": {
- "type": "object",
- "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "markdown"
- },
- "value": {
- "$ref": "#/definitions/text_value",
- "examples": [
- "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
- ]
- }
- }
- },
- "commit": {
- "type": "object",
- "description": "A commit/tag/branch within the GitLab project",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "commit"
- },
- "value": {
- "type": "string",
- "description": "The commit SHA",
- "minLength": 1
- }
- }
- },
- "file_location": {
- "type": "object",
- "description": "A location within a file in the project",
- "required": [
- "type",
- "file_name",
- "line_start"
- ],
- "properties": {
- "type": {
- "const": "file-location"
- },
- "file_name": {
- "type": "string",
- "minLength": 1
- },
- "line_start": {
- "type": "integer"
- },
- "line_end": {
- "type": "integer"
- }
- }
- },
- "module_location": {
- "type": "object",
- "description": "A location within a binary module of the form module+relative_offset",
- "required": [
- "type",
- "module_name",
- "offset"
- ],
- "properties": {
- "type": {
- "const": "module-location"
- },
- "module_name": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "compiled_binary"
- ]
- },
- "offset": {
- "type": "integer",
- "examples": [
- 100
- ]
- }
- }
- }
- },
- "self": {
- "version": "14.0.1"
- },
- "required": [
- "version",
- "vulnerabilities"
- ],
- "additionalProperties": true,
- "properties": {
- "scan": {
- "type": "object",
- "required": [
- "end_time",
- "scanner",
- "start_time",
- "status",
- "type"
- ],
- "properties": {
- "end_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-01-28T03:26:02"
- ]
- },
- "messages": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Communication intended for the initiator of a scan.",
- "required": [
- "level",
- "value"
- ],
- "properties": {
- "level": {
- "type": "string",
- "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
- "enum": [
- "info",
- "warn",
- "fatal"
- ],
- "examples": [
- "info"
- ]
- },
- "value": {
- "type": "string",
- "description": "The message to communicate.",
- "minLength": 1,
- "examples": [
- "Permission denied, scanning aborted"
- ]
- }
- }
- }
- },
- "scanner": {
- "type": "object",
- "description": "Object defining the scanner used to perform the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the scanner.",
- "minLength": 1,
- "examples": [
- "my-sast-scanner"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the scanner, not required to be unique.",
- "minLength": 1,
- "examples": [
- "My SAST Scanner"
- ]
- },
- "url": {
- "type": "string",
- "description": "A link to more information about the scanner.",
- "examples": [
- "https://scanner.url"
- ]
- },
- "version": {
- "type": "string",
- "description": "The version of the scanner.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- },
- "vendor": {
- "type": "object",
- "description": "The vendor/maintainer of the scanner.",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- }
- }
- },
- "start_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-02-14T16:01:59"
- ]
- },
- "status": {
- "type": "string",
- "description": "Result of the scan.",
- "enum": [
- "success",
- "failure"
- ]
- },
- "type": {
- "type": "string",
- "description": "Type of the scan.",
- "enum": [
- "container_scanning"
- ]
- }
- }
- },
- "schema": {
- "type": "string",
- "description": "URI pointing to the validating security report schema.",
- "format": "uri"
- },
- "version": {
- "type": "string",
- "description": "The version of the schema to which the JSON report conforms.",
- "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
- },
- "vulnerabilities": {
- "type": "array",
- "description": "Array of vulnerability objects.",
- "items": {
- "type": "object",
- "description": "Describes the vulnerability.",
- "required": [
- "category",
- "cve",
- "identifiers",
- "location",
- "scanner"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
- "examples": [
- "642735a5-1425-428d-8d4e-3c854885a3c9"
- ]
- },
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
- "name": {
- "type": "string",
- "description": "The name of the vulnerability. This must not include the finding's specific information."
- },
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
- "description": {
- "type": "string",
- "description": "A long text section describing the vulnerability more fully."
- },
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
- "severity": {
- "type": "string",
- "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Info",
- "Unknown",
- "Low",
- "Medium",
- "High",
- "Critical"
- ]
- },
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
- "solution": {
- "type": "string",
- "description": "Explanation of how to fix the vulnerability."
- },
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
- "identifiers": {
- "type": "array",
- "minItems": 1,
- "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
- "items": {
- "type": "object",
- "required": [
- "type",
- "name",
- "value"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
- "minLength": 1
- },
- "name": {
- "type": "string",
- "description": "Human-readable name of the identifier.",
- "minLength": 1
- },
- "url": {
- "type": "string",
- "description": "URL of the identifier's documentation.",
- "format": "uri"
- },
- "value": {
- "type": "string",
- "description": "Value of the identifier, for matching purpose.",
- "minLength": 1
- }
- }
- }
- },
- "links": {
- "type": "array",
- "description": "An array of references to external documentation or articles that describe the vulnerability.",
- "items": {
- "type": "object",
- "required": [
- "url"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the vulnerability details link."
- },
- "url": {
- "type": "string",
- "description": "URL of the vulnerability details document.",
- "format": "uri"
- }
- }
- }
- },
- "details": {
- "$ref": "#/definitions/named_list/properties/items"
- },
- "tracking": {
- "description": "Describes how this vulnerability should be tracked as the project changes.",
- "oneOf": [
- {
- "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
- "required": [
- "items"
- ],
- "properties": {
- "type": {
- "const": "source"
- },
- "items": {
- "type": "array",
- "items": {
- "description": "An item that should be tracked using source-specific tracking methods.",
- "type": "object",
- "required": [
- "signatures"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the file that includes the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the file that includes the vulnerability."
- },
- "signatures": {
- "type": "array",
- "description": "An array of calculated tracking signatures for this tracking item.",
- "minItems": 1,
- "items": {
- "description": "A calculated tracking signature value and metadata.",
- "required": [
- "algorithm",
- "value"
- ],
- "properties": {
- "algorithm": {
- "type": "string",
- "description": "The algorithm used to generate the signature."
- },
- "value": {
- "type": "string",
- "description": "The result of this signature algorithm."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "Each tracking type must declare its own type."
- }
- }
- },
- "location": {
- "type": "object",
- "description": "Identifies the vulnerability's location.",
- "required": [
- "dependency",
- "operating_system",
- "image"
- ],
- "properties": {
- "dependency": {
- "type": "object",
- "description": "Describes the dependency of a project where the vulnerability is located.",
- "properties": {
- "package": {
- "type": "object",
- "description": "Provides information on the package where the vulnerability is located.",
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the package where the vulnerability is located."
- }
- }
- },
- "version": {
- "type": "string",
- "description": "Version of the vulnerable package."
- },
- "iid": {
- "description": "ID that identifies the dependency in the scope of a dependency file.",
- "type": "number"
- },
- "direct": {
- "type": "boolean",
- "description": "Tells whether this is a direct, top-level dependency of the scanned project."
- },
- "dependency_path": {
- "type": "array",
- "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.",
- "items": {
- "type": "object",
- "required": [
- "iid"
- ],
- "properties": {
- "iid": {
- "type": "number",
- "description": "ID that is unique in the scope of a parent object, and specific to the resource type."
- }
- }
- }
- }
- }
- },
- "operating_system": {
- "type": "string",
- "minLength": 1,
- "description": "The operating system that contains the vulnerable package."
- },
- "image": {
- "type": "string",
- "minLength": 1,
- "description": "The analyzed Docker image."
- }
- }
- }
- }
- }
- },
- "remediations": {
- "type": "array",
- "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
- "items": {
- "type": "object",
- "required": [
- "fixes",
- "summary",
- "diff"
- ],
- "properties": {
- "fixes": {
- "type": "array",
- "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
- "items": {
- "type": "object",
- "required": [
- "cve"
- ],
- "properties": {
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- }
- }
- }
- },
- "summary": {
- "type": "string",
- "minLength": 1,
- "description": "An overview of how the vulnerabilities were fixed."
- },
- "diff": {
- "type": "string",
- "minLength": 1,
- "description": "A base64-encoded remediation code diff, compatible with git apply."
- }
- }
- }
- }
- }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.1/coverage-fuzzing-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.1/coverage-fuzzing-report-format.json
deleted file mode 100644
index d496b62ee7f..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.1/coverage-fuzzing-report-format.json
+++ /dev/null
@@ -1,779 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Report format for GitLab Fuzz Testing",
- "description": "This schema provides the report format for Coverage Guided Fuzz Testing (https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing).",
- "definitions": {
- "detail_type": {
- "oneOf": [
- {
- "$ref": "#/definitions/named_list"
- },
- {
- "$ref": "#/definitions/list"
- },
- {
- "$ref": "#/definitions/table"
- },
- {
- "$ref": "#/definitions/text"
- },
- {
- "$ref": "#/definitions/url"
- },
- {
- "$ref": "#/definitions/code"
- },
- {
- "$ref": "#/definitions/value"
- },
- {
- "$ref": "#/definitions/diff"
- },
- {
- "$ref": "#/definitions/markdown"
- },
- {
- "$ref": "#/definitions/commit"
- },
- {
- "$ref": "#/definitions/file_location"
- },
- {
- "$ref": "#/definitions/module_location"
- }
- ]
- },
- "text_value": {
- "type": "string"
- },
- "named_field": {
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "$ref": "#/definitions/text_value",
- "minLength": 1
- },
- "description": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "named_list": {
- "type": "object",
- "description": "An object with named and typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "named-list"
- },
- "items": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- {
- "$ref": "#/definitions/named_field"
- },
- {
- "$ref": "#/definitions/detail_type"
- }
- ]
- }
- }
- }
- }
- },
- "list": {
- "type": "object",
- "description": "A list of typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- },
- "table": {
- "type": "object",
- "description": "A table of typed fields",
- "required": [
- "type",
- "rows"
- ],
- "properties": {
- "type": {
- "const": "table"
- },
- "header": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- },
- "rows": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- }
- },
- "text": {
- "type": "object",
- "description": "Raw text",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "text"
- },
- "value": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "url": {
- "type": "object",
- "description": "A single URL",
- "required": [
- "type",
- "href"
- ],
- "properties": {
- "type": {
- "const": "url"
- },
- "text": {
- "$ref": "#/definitions/text_value"
- },
- "href": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "http://mysite.com"
- ]
- }
- }
- },
- "code": {
- "type": "object",
- "description": "A codeblock",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "code"
- },
- "value": {
- "type": "string"
- },
- "lang": {
- "type": "string",
- "description": "A programming language"
- }
- }
- },
- "value": {
- "type": "object",
- "description": "A field that can store a range of types of value",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "value"
- },
- "value": {
- "type": [
- "number",
- "string",
- "boolean"
- ]
- }
- }
- },
- "diff": {
- "type": "object",
- "description": "A diff",
- "required": [
- "type",
- "before",
- "after"
- ],
- "properties": {
- "type": {
- "const": "diff"
- },
- "before": {
- "type": "string"
- },
- "after": {
- "type": "string"
- }
- }
- },
- "markdown": {
- "type": "object",
- "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "markdown"
- },
- "value": {
- "$ref": "#/definitions/text_value",
- "examples": [
- "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
- ]
- }
- }
- },
- "commit": {
- "type": "object",
- "description": "A commit/tag/branch within the GitLab project",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "commit"
- },
- "value": {
- "type": "string",
- "description": "The commit SHA",
- "minLength": 1
- }
- }
- },
- "file_location": {
- "type": "object",
- "description": "A location within a file in the project",
- "required": [
- "type",
- "file_name",
- "line_start"
- ],
- "properties": {
- "type": {
- "const": "file-location"
- },
- "file_name": {
- "type": "string",
- "minLength": 1
- },
- "line_start": {
- "type": "integer"
- },
- "line_end": {
- "type": "integer"
- }
- }
- },
- "module_location": {
- "type": "object",
- "description": "A location within a binary module of the form module+relative_offset",
- "required": [
- "type",
- "module_name",
- "offset"
- ],
- "properties": {
- "type": {
- "const": "module-location"
- },
- "module_name": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "compiled_binary"
- ]
- },
- "offset": {
- "type": "integer",
- "examples": [
- 100
- ]
- }
- }
- }
- },
- "self": {
- "version": "14.0.1"
- },
- "required": [
- "version",
- "vulnerabilities"
- ],
- "additionalProperties": true,
- "properties": {
- "scan": {
- "type": "object",
- "required": [
- "end_time",
- "scanner",
- "start_time",
- "status",
- "type"
- ],
- "properties": {
- "end_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-01-28T03:26:02"
- ]
- },
- "messages": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Communication intended for the initiator of a scan.",
- "required": [
- "level",
- "value"
- ],
- "properties": {
- "level": {
- "type": "string",
- "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
- "enum": [
- "info",
- "warn",
- "fatal"
- ],
- "examples": [
- "info"
- ]
- },
- "value": {
- "type": "string",
- "description": "The message to communicate.",
- "minLength": 1,
- "examples": [
- "Permission denied, scanning aborted"
- ]
- }
- }
- }
- },
- "scanner": {
- "type": "object",
- "description": "Object defining the scanner used to perform the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the scanner.",
- "minLength": 1,
- "examples": [
- "my-sast-scanner"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the scanner, not required to be unique.",
- "minLength": 1,
- "examples": [
- "My SAST Scanner"
- ]
- },
- "url": {
- "type": "string",
- "description": "A link to more information about the scanner.",
- "examples": [
- "https://scanner.url"
- ]
- },
- "version": {
- "type": "string",
- "description": "The version of the scanner.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- },
- "vendor": {
- "type": "object",
- "description": "The vendor/maintainer of the scanner.",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- }
- }
- },
- "start_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-02-14T16:01:59"
- ]
- },
- "status": {
- "type": "string",
- "description": "Result of the scan.",
- "enum": [
- "success",
- "failure"
- ]
- },
- "type": {
- "type": "string",
- "description": "Type of the scan.",
- "enum": [
- "coverage_fuzzing"
- ]
- }
- }
- },
- "schema": {
- "type": "string",
- "description": "URI pointing to the validating security report schema.",
- "format": "uri"
- },
- "version": {
- "type": "string",
- "description": "The version of the schema to which the JSON report conforms.",
- "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
- },
- "vulnerabilities": {
- "type": "array",
- "description": "Array of vulnerability objects.",
- "items": {
- "type": "object",
- "description": "Describes the vulnerability.",
- "required": [
- "category",
- "cve",
- "identifiers",
- "location",
- "scanner"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
- "examples": [
- "642735a5-1425-428d-8d4e-3c854885a3c9"
- ]
- },
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
- "name": {
- "type": "string",
- "description": "The name of the vulnerability. This must not include the finding's specific information."
- },
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
- "description": {
- "type": "string",
- "description": "A long text section describing the vulnerability more fully."
- },
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
- "severity": {
- "type": "string",
- "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Info",
- "Unknown",
- "Low",
- "Medium",
- "High",
- "Critical"
- ]
- },
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
- "solution": {
- "type": "string",
- "description": "Explanation of how to fix the vulnerability."
- },
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
- "identifiers": {
- "type": "array",
- "minItems": 1,
- "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
- "items": {
- "type": "object",
- "required": [
- "type",
- "name",
- "value"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
- "minLength": 1
- },
- "name": {
- "type": "string",
- "description": "Human-readable name of the identifier.",
- "minLength": 1
- },
- "url": {
- "type": "string",
- "description": "URL of the identifier's documentation.",
- "format": "uri"
- },
- "value": {
- "type": "string",
- "description": "Value of the identifier, for matching purpose.",
- "minLength": 1
- }
- }
- }
- },
- "links": {
- "type": "array",
- "description": "An array of references to external documentation or articles that describe the vulnerability.",
- "items": {
- "type": "object",
- "required": [
- "url"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the vulnerability details link."
- },
- "url": {
- "type": "string",
- "description": "URL of the vulnerability details document.",
- "format": "uri"
- }
- }
- }
- },
- "details": {
- "$ref": "#/definitions/named_list/properties/items"
- },
- "tracking": {
- "description": "Describes how this vulnerability should be tracked as the project changes.",
- "oneOf": [
- {
- "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
- "required": [
- "items"
- ],
- "properties": {
- "type": {
- "const": "source"
- },
- "items": {
- "type": "array",
- "items": {
- "description": "An item that should be tracked using source-specific tracking methods.",
- "type": "object",
- "required": [
- "signatures"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the file that includes the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the file that includes the vulnerability."
- },
- "signatures": {
- "type": "array",
- "description": "An array of calculated tracking signatures for this tracking item.",
- "minItems": 1,
- "items": {
- "description": "A calculated tracking signature value and metadata.",
- "required": [
- "algorithm",
- "value"
- ],
- "properties": {
- "algorithm": {
- "type": "string",
- "description": "The algorithm used to generate the signature."
- },
- "value": {
- "type": "string",
- "description": "The result of this signature algorithm."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "Each tracking type must declare its own type."
- }
- }
- },
- "location": {
- "description": "The location of the error",
- "type": "object",
- "properties": {
- "crash_address": {
- "type": "string",
- "description": "The relative address in memory were the crash occurred.",
- "examples": [
- "0xabababab"
- ]
- },
- "stacktrace_snippet": {
- "type": "string",
- "description": "The stack trace recorded during fuzzing resulting the crash.",
- "examples": [
- "func_a+0xabcd\nfunc_b+0xabcc"
- ]
- },
- "crash_state": {
- "type": "string",
- "description": "Minimised and normalized crash stack-trace (called crash_state).",
- "examples": [
- "func_a+0xa\nfunc_b+0xb\nfunc_c+0xc"
- ]
- },
- "crash_type": {
- "type": "string",
- "description": "Type of the crash.",
- "examples": [
- "Heap-Buffer-overflow",
- "Division-by-zero"
- ]
- }
- }
- }
- }
- }
- },
- "remediations": {
- "type": "array",
- "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
- "items": {
- "type": "object",
- "required": [
- "fixes",
- "summary",
- "diff"
- ],
- "properties": {
- "fixes": {
- "type": "array",
- "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
- "items": {
- "type": "object",
- "required": [
- "cve"
- ],
- "properties": {
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- }
- }
- }
- },
- "summary": {
- "type": "string",
- "minLength": 1,
- "description": "An overview of how the vulnerabilities were fixed."
- },
- "diff": {
- "type": "string",
- "minLength": 1,
- "description": "A base64-encoded remediation code diff, compatible with git apply."
- }
- }
- }
- }
- }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.1/dast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.1/dast-report-format.json
deleted file mode 100644
index a4d59f39a15..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.1/dast-report-format.json
+++ /dev/null
@@ -1,1196 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Report format for GitLab DAST",
- "description": "This schema provides the the report format for Dynamic Application Security Testing (https://docs.gitlab.com/ee/user/application_security/dast).",
- "definitions": {
- "detail_type": {
- "oneOf": [
- {
- "$ref": "#/definitions/named_list"
- },
- {
- "$ref": "#/definitions/list"
- },
- {
- "$ref": "#/definitions/table"
- },
- {
- "$ref": "#/definitions/text"
- },
- {
- "$ref": "#/definitions/url"
- },
- {
- "$ref": "#/definitions/code"
- },
- {
- "$ref": "#/definitions/value"
- },
- {
- "$ref": "#/definitions/diff"
- },
- {
- "$ref": "#/definitions/markdown"
- },
- {
- "$ref": "#/definitions/commit"
- },
- {
- "$ref": "#/definitions/file_location"
- },
- {
- "$ref": "#/definitions/module_location"
- }
- ]
- },
- "text_value": {
- "type": "string"
- },
- "named_field": {
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "$ref": "#/definitions/text_value",
- "minLength": 1
- },
- "description": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "named_list": {
- "type": "object",
- "description": "An object with named and typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "named-list"
- },
- "items": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- {
- "$ref": "#/definitions/named_field"
- },
- {
- "$ref": "#/definitions/detail_type"
- }
- ]
- }
- }
- }
- }
- },
- "list": {
- "type": "object",
- "description": "A list of typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- },
- "table": {
- "type": "object",
- "description": "A table of typed fields",
- "required": [
- "type",
- "rows"
- ],
- "properties": {
- "type": {
- "const": "table"
- },
- "header": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- },
- "rows": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- }
- },
- "text": {
- "type": "object",
- "description": "Raw text",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "text"
- },
- "value": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "url": {
- "type": "object",
- "description": "A single URL",
- "required": [
- "type",
- "href"
- ],
- "properties": {
- "type": {
- "const": "url"
- },
- "text": {
- "$ref": "#/definitions/text_value"
- },
- "href": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "http://mysite.com"
- ]
- }
- }
- },
- "code": {
- "type": "object",
- "description": "A codeblock",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "code"
- },
- "value": {
- "type": "string"
- },
- "lang": {
- "type": "string",
- "description": "A programming language"
- }
- }
- },
- "value": {
- "type": "object",
- "description": "A field that can store a range of types of value",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "value"
- },
- "value": {
- "type": [
- "number",
- "string",
- "boolean"
- ]
- }
- }
- },
- "diff": {
- "type": "object",
- "description": "A diff",
- "required": [
- "type",
- "before",
- "after"
- ],
- "properties": {
- "type": {
- "const": "diff"
- },
- "before": {
- "type": "string"
- },
- "after": {
- "type": "string"
- }
- }
- },
- "markdown": {
- "type": "object",
- "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "markdown"
- },
- "value": {
- "$ref": "#/definitions/text_value",
- "examples": [
- "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
- ]
- }
- }
- },
- "commit": {
- "type": "object",
- "description": "A commit/tag/branch within the GitLab project",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "commit"
- },
- "value": {
- "type": "string",
- "description": "The commit SHA",
- "minLength": 1
- }
- }
- },
- "file_location": {
- "type": "object",
- "description": "A location within a file in the project",
- "required": [
- "type",
- "file_name",
- "line_start"
- ],
- "properties": {
- "type": {
- "const": "file-location"
- },
- "file_name": {
- "type": "string",
- "minLength": 1
- },
- "line_start": {
- "type": "integer"
- },
- "line_end": {
- "type": "integer"
- }
- }
- },
- "module_location": {
- "type": "object",
- "description": "A location within a binary module of the form module+relative_offset",
- "required": [
- "type",
- "module_name",
- "offset"
- ],
- "properties": {
- "type": {
- "const": "module-location"
- },
- "module_name": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "compiled_binary"
- ]
- },
- "offset": {
- "type": "integer",
- "examples": [
- 100
- ]
- }
- }
- }
- },
- "self": {
- "version": "14.0.1"
- },
- "required": [
- "version",
- "vulnerabilities"
- ],
- "additionalProperties": true,
- "properties": {
- "scan": {
- "type": "object",
- "required": [
- "end_time",
- "scanned_resources",
- "scanner",
- "start_time",
- "status",
- "type"
- ],
- "properties": {
- "end_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-01-28T03:26:02"
- ]
- },
- "messages": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Communication intended for the initiator of a scan.",
- "required": [
- "level",
- "value"
- ],
- "properties": {
- "level": {
- "type": "string",
- "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
- "enum": [
- "info",
- "warn",
- "fatal"
- ],
- "examples": [
- "info"
- ]
- },
- "value": {
- "type": "string",
- "description": "The message to communicate.",
- "minLength": 1,
- "examples": [
- "Permission denied, scanning aborted"
- ]
- }
- }
- }
- },
- "scanner": {
- "type": "object",
- "description": "Object defining the scanner used to perform the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the scanner.",
- "minLength": 1,
- "examples": [
- "my-sast-scanner"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the scanner, not required to be unique.",
- "minLength": 1,
- "examples": [
- "My SAST Scanner"
- ]
- },
- "url": {
- "type": "string",
- "description": "A link to more information about the scanner.",
- "examples": [
- "https://scanner.url"
- ]
- },
- "version": {
- "type": "string",
- "description": "The version of the scanner.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- },
- "vendor": {
- "type": "object",
- "description": "The vendor/maintainer of the scanner.",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- }
- }
- },
- "start_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-02-14T16:01:59"
- ]
- },
- "status": {
- "type": "string",
- "description": "Result of the scan.",
- "enum": [
- "success",
- "failure"
- ]
- },
- "type": {
- "type": "string",
- "description": "Type of the scan.",
- "enum": [
- "dast",
- "api_fuzzing"
- ]
- },
- "scanned_resources": {
- "type": "array",
- "description": "The attack surface scanned by DAST.",
- "items": {
- "type": "object",
- "required": [
- "method",
- "url",
- "type"
- ],
- "properties": {
- "method": {
- "type": "string",
- "minLength": 1,
- "description": "HTTP method of the scanned resource.",
- "examples": [
- "GET",
- "POST",
- "HEAD"
- ]
- },
- "url": {
- "type": "string",
- "minLength": 1,
- "description": "URL of the scanned resource.",
- "examples": [
- "http://my.site.com/a-page"
- ]
- },
- "type": {
- "type": "string",
- "minLength": 1,
- "description": "Type of the scanned resource, for DAST, this must be 'url'.",
- "examples": [
- "url"
- ]
- }
- }
- }
- }
- }
- },
- "schema": {
- "type": "string",
- "description": "URI pointing to the validating security report schema.",
- "format": "uri"
- },
- "version": {
- "type": "string",
- "description": "The version of the schema to which the JSON report conforms.",
- "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
- },
- "vulnerabilities": {
- "type": "array",
- "description": "Array of vulnerability objects.",
- "items": {
- "type": "object",
- "description": "Describes the vulnerability.",
- "required": [
- "category",
- "cve",
- "identifiers",
- "location",
- "scanner"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
- "examples": [
- "642735a5-1425-428d-8d4e-3c854885a3c9"
- ]
- },
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
- "name": {
- "type": "string",
- "description": "The name of the vulnerability. This must not include the finding's specific information."
- },
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
- "description": {
- "type": "string",
- "description": "A long text section describing the vulnerability more fully."
- },
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
- "severity": {
- "type": "string",
- "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Info",
- "Unknown",
- "Low",
- "Medium",
- "High",
- "Critical"
- ]
- },
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
- "solution": {
- "type": "string",
- "description": "Explanation of how to fix the vulnerability."
- },
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
- "identifiers": {
- "type": "array",
- "minItems": 1,
- "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
- "items": {
- "type": "object",
- "required": [
- "type",
- "name",
- "value"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
- "minLength": 1
- },
- "name": {
- "type": "string",
- "description": "Human-readable name of the identifier.",
- "minLength": 1
- },
- "url": {
- "type": "string",
- "description": "URL of the identifier's documentation.",
- "format": "uri"
- },
- "value": {
- "type": "string",
- "description": "Value of the identifier, for matching purpose.",
- "minLength": 1
- }
- }
- }
- },
- "links": {
- "type": "array",
- "description": "An array of references to external documentation or articles that describe the vulnerability.",
- "items": {
- "type": "object",
- "required": [
- "url"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the vulnerability details link."
- },
- "url": {
- "type": "string",
- "description": "URL of the vulnerability details document.",
- "format": "uri"
- }
- }
- }
- },
- "details": {
- "$ref": "#/definitions/named_list/properties/items"
- },
- "tracking": {
- "description": "Describes how this vulnerability should be tracked as the project changes.",
- "oneOf": [
- {
- "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
- "required": [
- "items"
- ],
- "properties": {
- "type": {
- "const": "source"
- },
- "items": {
- "type": "array",
- "items": {
- "description": "An item that should be tracked using source-specific tracking methods.",
- "type": "object",
- "required": [
- "signatures"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the file that includes the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the file that includes the vulnerability."
- },
- "signatures": {
- "type": "array",
- "description": "An array of calculated tracking signatures for this tracking item.",
- "minItems": 1,
- "items": {
- "description": "A calculated tracking signature value and metadata.",
- "required": [
- "algorithm",
- "value"
- ],
- "properties": {
- "algorithm": {
- "type": "string",
- "description": "The algorithm used to generate the signature."
- },
- "value": {
- "type": "string",
- "description": "The result of this signature algorithm."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "Each tracking type must declare its own type."
- }
- }
- },
- "evidence": {
- "type": "object",
- "properties": {
- "source": {
- "type": "object",
- "description": "Source of evidence",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "Unique source identifier",
- "examples": [
- "assert:LogAnalysis",
- "assert:StatusCode"
- ]
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Source display name",
- "examples": [
- "Log Analysis",
- "Status Code"
- ]
- },
- "url": {
- "type": "string",
- "description": "Link to additional information",
- "examples": [
- "https://docs.gitlab.com/ee/development/integrations/secure.html"
- ]
- }
- }
- },
- "summary": {
- "type": "string",
- "description": "Human readable string containing evidence of the vulnerability.",
- "examples": [
- "Credit card 4111111111111111 found",
- "Server leaked information nginx/1.17.6"
- ]
- },
- "request": {
- "type": "object",
- "description": "An HTTP request.",
- "required": [
- "headers",
- "method",
- "url"
- ],
- "properties": {
- "headers": {
- "type": "array",
- "description": "HTTP headers present on the request.",
- "items": {
- "type": "object",
- "required": [
- "name",
- "value"
- ],
- "properties": {
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Name of the HTTP header.",
- "examples": [
- "Accept",
- "Content-Length",
- "Content-Type"
- ]
- },
- "value": {
- "type": "string",
- "minLength": 1,
- "description": "Value of the HTTP header.",
- "examples": [
- "*/*",
- "560",
- "application/json; charset=utf-8"
- ]
- }
- }
- }
- },
- "method": {
- "type": "string",
- "minLength": 1,
- "description": "HTTP method used in the request.",
- "examples": [
- "GET",
- "POST"
- ]
- },
- "url": {
- "type": "string",
- "minLength": 1,
- "description": "URL of the request.",
- "examples": [
- "http://my.site.com/vulnerable-endpoint?show-credit-card"
- ]
- },
- "body": {
- "type": "string",
- "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.",
- "examples": [
- "user=jsmith&first=%27&last=smith"
- ]
- }
- }
- },
- "response": {
- "type": "object",
- "description": "An HTTP response.",
- "required": [
- "headers",
- "reason_phrase",
- "status_code"
- ],
- "properties": {
- "headers": {
- "type": "array",
- "description": "HTTP headers present on the request.",
- "items": {
- "type": "object",
- "required": [
- "name",
- "value"
- ],
- "properties": {
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Name of the HTTP header.",
- "examples": [
- "Accept",
- "Content-Length",
- "Content-Type"
- ]
- },
- "value": {
- "type": "string",
- "minLength": 1,
- "description": "Value of the HTTP header.",
- "examples": [
- "*/*",
- "560",
- "application/json; charset=utf-8"
- ]
- }
- }
- }
- },
- "reason_phrase": {
- "type": "string",
- "description": "HTTP reason phrase of the response.",
- "examples": [
- "OK",
- "Internal Server Error"
- ]
- },
- "status_code": {
- "type": "integer",
- "description": "HTTP status code of the response.",
- "examples": [
- 200,
- 500
- ]
- },
- "body": {
- "type": "string",
- "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.",
- "examples": [
- "{\"user_id\": 2}"
- ]
- }
- }
- },
- "supporting_messages": {
- "type": "array",
- "description": "Array of supporting http messages.",
- "items": {
- "type": "object",
- "description": "A supporting http message.",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Message display name.",
- "examples": [
- "Unmodified",
- "Recorded"
- ]
- },
- "request": {
- "type": "object",
- "description": "An HTTP request.",
- "required": [
- "headers",
- "method",
- "url"
- ],
- "properties": {
- "headers": {
- "type": "array",
- "description": "HTTP headers present on the request.",
- "items": {
- "type": "object",
- "required": [
- "name",
- "value"
- ],
- "properties": {
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Name of the HTTP header.",
- "examples": [
- "Accept",
- "Content-Length",
- "Content-Type"
- ]
- },
- "value": {
- "type": "string",
- "minLength": 1,
- "description": "Value of the HTTP header.",
- "examples": [
- "*/*",
- "560",
- "application/json; charset=utf-8"
- ]
- }
- }
- }
- },
- "method": {
- "type": "string",
- "minLength": 1,
- "description": "HTTP method used in the request.",
- "examples": [
- "GET",
- "POST"
- ]
- },
- "url": {
- "type": "string",
- "minLength": 1,
- "description": "URL of the request.",
- "examples": [
- "http://my.site.com/vulnerable-endpoint?show-credit-card"
- ]
- },
- "body": {
- "type": "string",
- "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.",
- "examples": [
- "user=jsmith&first=%27&last=smith"
- ]
- }
- }
- },
- "response": {
- "type": "object",
- "description": "An HTTP response.",
- "required": [
- "headers",
- "reason_phrase",
- "status_code"
- ],
- "properties": {
- "headers": {
- "type": "array",
- "description": "HTTP headers present on the request.",
- "items": {
- "type": "object",
- "required": [
- "name",
- "value"
- ],
- "properties": {
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Name of the HTTP header.",
- "examples": [
- "Accept",
- "Content-Length",
- "Content-Type"
- ]
- },
- "value": {
- "type": "string",
- "minLength": 1,
- "description": "Value of the HTTP header.",
- "examples": [
- "*/*",
- "560",
- "application/json; charset=utf-8"
- ]
- }
- }
- }
- },
- "reason_phrase": {
- "type": "string",
- "description": "HTTP reason phrase of the response.",
- "examples": [
- "OK",
- "Internal Server Error"
- ]
- },
- "status_code": {
- "type": "integer",
- "description": "HTTP status code of the response.",
- "examples": [
- 200,
- 500
- ]
- },
- "body": {
- "type": "string",
- "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.",
- "examples": [
- "{\"user_id\": 2}"
- ]
- }
- }
- }
- }
- }
- }
- }
- },
- "location": {
- "type": "object",
- "description": "Identifies the vulnerability's location.",
- "properties": {
- "hostname": {
- "type": "string",
- "description": "The protocol, domain, and port of the application where the vulnerability was found."
- },
- "method": {
- "type": "string",
- "description": "The HTTP method that was used to request the URL where the vulnerability was found."
- },
- "param": {
- "type": "string",
- "description": "A value provided by a vulnerability rule related to the found vulnerability. Examples include a header value, or a parameter used in a HTTP POST."
- },
- "path": {
- "type": "string",
- "description": "The path of the URL where the vulnerability was found. Typically, this would start with a forward slash."
- }
- }
- },
- "assets": {
- "type": "array",
- "description": "Array of build assets associated with vulnerability.",
- "items": {
- "type": "object",
- "description": "Describes an asset associated with vulnerability.",
- "required": [
- "type",
- "name",
- "url"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "The type of asset",
- "enum": [
- "http_session",
- "postman"
- ]
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Display name for asset",
- "examples": [
- "HTTP Messages",
- "Postman Collection"
- ]
- },
- "url": {
- "type": "string",
- "minLength": 1,
- "description": "Link to asset in build artifacts",
- "examples": [
- "https://gitlab.com/gitlab-org/security-products/dast/-/jobs/626397001/artifacts/file//output/zap_session.data"
- ]
- }
- }
- }
- },
- "discovered_at": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss.sss, representing when the vulnerability was discovered",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}\\.\\d{3}$",
- "examples": [
- "2020-01-28T03:26:02.956"
- ]
- }
- }
- }
- },
- "remediations": {
- "type": "array",
- "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
- "items": {
- "type": "object",
- "required": [
- "fixes",
- "summary",
- "diff"
- ],
- "properties": {
- "fixes": {
- "type": "array",
- "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
- "items": {
- "type": "object",
- "required": [
- "cve"
- ],
- "properties": {
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- }
- }
- }
- },
- "summary": {
- "type": "string",
- "minLength": 1,
- "description": "An overview of how the vulnerabilities were fixed."
- },
- "diff": {
- "type": "string",
- "minLength": 1,
- "description": "A base64-encoded remediation code diff, compatible with git apply."
- }
- }
- }
- }
- }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.1/dependency-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.1/dependency-scanning-report-format.json
deleted file mode 100644
index c83d5195be4..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.1/dependency-scanning-report-format.json
+++ /dev/null
@@ -1,873 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Report format for GitLab Dependency Scanning",
- "description": "This schema provides the the report format for Dependency Scanning analyzers (https://docs.gitlab.com/ee/user/application_security/dependency_scanning).",
- "definitions": {
- "detail_type": {
- "oneOf": [
- {
- "$ref": "#/definitions/named_list"
- },
- {
- "$ref": "#/definitions/list"
- },
- {
- "$ref": "#/definitions/table"
- },
- {
- "$ref": "#/definitions/text"
- },
- {
- "$ref": "#/definitions/url"
- },
- {
- "$ref": "#/definitions/code"
- },
- {
- "$ref": "#/definitions/value"
- },
- {
- "$ref": "#/definitions/diff"
- },
- {
- "$ref": "#/definitions/markdown"
- },
- {
- "$ref": "#/definitions/commit"
- },
- {
- "$ref": "#/definitions/file_location"
- },
- {
- "$ref": "#/definitions/module_location"
- }
- ]
- },
- "text_value": {
- "type": "string"
- },
- "named_field": {
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "$ref": "#/definitions/text_value",
- "minLength": 1
- },
- "description": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "named_list": {
- "type": "object",
- "description": "An object with named and typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "named-list"
- },
- "items": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- {
- "$ref": "#/definitions/named_field"
- },
- {
- "$ref": "#/definitions/detail_type"
- }
- ]
- }
- }
- }
- }
- },
- "list": {
- "type": "object",
- "description": "A list of typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- },
- "table": {
- "type": "object",
- "description": "A table of typed fields",
- "required": [
- "type",
- "rows"
- ],
- "properties": {
- "type": {
- "const": "table"
- },
- "header": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- },
- "rows": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- }
- },
- "text": {
- "type": "object",
- "description": "Raw text",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "text"
- },
- "value": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "url": {
- "type": "object",
- "description": "A single URL",
- "required": [
- "type",
- "href"
- ],
- "properties": {
- "type": {
- "const": "url"
- },
- "text": {
- "$ref": "#/definitions/text_value"
- },
- "href": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "http://mysite.com"
- ]
- }
- }
- },
- "code": {
- "type": "object",
- "description": "A codeblock",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "code"
- },
- "value": {
- "type": "string"
- },
- "lang": {
- "type": "string",
- "description": "A programming language"
- }
- }
- },
- "value": {
- "type": "object",
- "description": "A field that can store a range of types of value",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "value"
- },
- "value": {
- "type": [
- "number",
- "string",
- "boolean"
- ]
- }
- }
- },
- "diff": {
- "type": "object",
- "description": "A diff",
- "required": [
- "type",
- "before",
- "after"
- ],
- "properties": {
- "type": {
- "const": "diff"
- },
- "before": {
- "type": "string"
- },
- "after": {
- "type": "string"
- }
- }
- },
- "markdown": {
- "type": "object",
- "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "markdown"
- },
- "value": {
- "$ref": "#/definitions/text_value",
- "examples": [
- "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
- ]
- }
- }
- },
- "commit": {
- "type": "object",
- "description": "A commit/tag/branch within the GitLab project",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "commit"
- },
- "value": {
- "type": "string",
- "description": "The commit SHA",
- "minLength": 1
- }
- }
- },
- "file_location": {
- "type": "object",
- "description": "A location within a file in the project",
- "required": [
- "type",
- "file_name",
- "line_start"
- ],
- "properties": {
- "type": {
- "const": "file-location"
- },
- "file_name": {
- "type": "string",
- "minLength": 1
- },
- "line_start": {
- "type": "integer"
- },
- "line_end": {
- "type": "integer"
- }
- }
- },
- "module_location": {
- "type": "object",
- "description": "A location within a binary module of the form module+relative_offset",
- "required": [
- "type",
- "module_name",
- "offset"
- ],
- "properties": {
- "type": {
- "const": "module-location"
- },
- "module_name": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "compiled_binary"
- ]
- },
- "offset": {
- "type": "integer",
- "examples": [
- 100
- ]
- }
- }
- }
- },
- "self": {
- "version": "14.0.1"
- },
- "required": [
- "dependency_files",
- "version",
- "vulnerabilities"
- ],
- "additionalProperties": true,
- "properties": {
- "scan": {
- "type": "object",
- "required": [
- "end_time",
- "scanner",
- "start_time",
- "status",
- "type"
- ],
- "properties": {
- "end_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-01-28T03:26:02"
- ]
- },
- "messages": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Communication intended for the initiator of a scan.",
- "required": [
- "level",
- "value"
- ],
- "properties": {
- "level": {
- "type": "string",
- "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
- "enum": [
- "info",
- "warn",
- "fatal"
- ],
- "examples": [
- "info"
- ]
- },
- "value": {
- "type": "string",
- "description": "The message to communicate.",
- "minLength": 1,
- "examples": [
- "Permission denied, scanning aborted"
- ]
- }
- }
- }
- },
- "scanner": {
- "type": "object",
- "description": "Object defining the scanner used to perform the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the scanner.",
- "minLength": 1,
- "examples": [
- "my-sast-scanner"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the scanner, not required to be unique.",
- "minLength": 1,
- "examples": [
- "My SAST Scanner"
- ]
- },
- "url": {
- "type": "string",
- "description": "A link to more information about the scanner.",
- "examples": [
- "https://scanner.url"
- ]
- },
- "version": {
- "type": "string",
- "description": "The version of the scanner.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- },
- "vendor": {
- "type": "object",
- "description": "The vendor/maintainer of the scanner.",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- }
- }
- },
- "start_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-02-14T16:01:59"
- ]
- },
- "status": {
- "type": "string",
- "description": "Result of the scan.",
- "enum": [
- "success",
- "failure"
- ]
- },
- "type": {
- "type": "string",
- "description": "Type of the scan.",
- "enum": [
- "dependency_scanning"
- ]
- }
- }
- },
- "schema": {
- "type": "string",
- "description": "URI pointing to the validating security report schema.",
- "format": "uri"
- },
- "version": {
- "type": "string",
- "description": "The version of the schema to which the JSON report conforms.",
- "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
- },
- "vulnerabilities": {
- "type": "array",
- "description": "Array of vulnerability objects.",
- "items": {
- "type": "object",
- "description": "Describes the vulnerability.",
- "required": [
- "category",
- "cve",
- "identifiers",
- "location",
- "scanner"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
- "examples": [
- "642735a5-1425-428d-8d4e-3c854885a3c9"
- ]
- },
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
- "name": {
- "type": "string",
- "description": "The name of the vulnerability. This must not include the finding's specific information."
- },
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
- "description": {
- "type": "string",
- "description": "A long text section describing the vulnerability more fully."
- },
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
- "severity": {
- "type": "string",
- "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Info",
- "Unknown",
- "Low",
- "Medium",
- "High",
- "Critical"
- ]
- },
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
- "solution": {
- "type": "string",
- "description": "Explanation of how to fix the vulnerability."
- },
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
- "identifiers": {
- "type": "array",
- "minItems": 1,
- "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
- "items": {
- "type": "object",
- "required": [
- "type",
- "name",
- "value"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
- "minLength": 1
- },
- "name": {
- "type": "string",
- "description": "Human-readable name of the identifier.",
- "minLength": 1
- },
- "url": {
- "type": "string",
- "description": "URL of the identifier's documentation.",
- "format": "uri"
- },
- "value": {
- "type": "string",
- "description": "Value of the identifier, for matching purpose.",
- "minLength": 1
- }
- }
- }
- },
- "links": {
- "type": "array",
- "description": "An array of references to external documentation or articles that describe the vulnerability.",
- "items": {
- "type": "object",
- "required": [
- "url"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the vulnerability details link."
- },
- "url": {
- "type": "string",
- "description": "URL of the vulnerability details document.",
- "format": "uri"
- }
- }
- }
- },
- "details": {
- "$ref": "#/definitions/named_list/properties/items"
- },
- "tracking": {
- "description": "Describes how this vulnerability should be tracked as the project changes.",
- "oneOf": [
- {
- "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
- "required": [
- "items"
- ],
- "properties": {
- "type": {
- "const": "source"
- },
- "items": {
- "type": "array",
- "items": {
- "description": "An item that should be tracked using source-specific tracking methods.",
- "type": "object",
- "required": [
- "signatures"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the file that includes the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the file that includes the vulnerability."
- },
- "signatures": {
- "type": "array",
- "description": "An array of calculated tracking signatures for this tracking item.",
- "minItems": 1,
- "items": {
- "description": "A calculated tracking signature value and metadata.",
- "required": [
- "algorithm",
- "value"
- ],
- "properties": {
- "algorithm": {
- "type": "string",
- "description": "The algorithm used to generate the signature."
- },
- "value": {
- "type": "string",
- "description": "The result of this signature algorithm."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "Each tracking type must declare its own type."
- }
- }
- },
- "location": {
- "type": "object",
- "description": "Identifies the vulnerability's location.",
- "required": [
- "file",
- "dependency"
- ],
- "properties": {
- "file": {
- "type": "string",
- "minLength": 1,
- "description": "Path to the manifest or lock file where the dependency is declared (such as yarn.lock)."
- },
- "dependency": {
- "type": "object",
- "description": "Describes the dependency of a project where the vulnerability is located.",
- "properties": {
- "package": {
- "type": "object",
- "description": "Provides information on the package where the vulnerability is located.",
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the package where the vulnerability is located."
- }
- }
- },
- "version": {
- "type": "string",
- "description": "Version of the vulnerable package."
- },
- "iid": {
- "description": "ID that identifies the dependency in the scope of a dependency file.",
- "type": "number"
- },
- "direct": {
- "type": "boolean",
- "description": "Tells whether this is a direct, top-level dependency of the scanned project."
- },
- "dependency_path": {
- "type": "array",
- "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.",
- "items": {
- "type": "object",
- "required": [
- "iid"
- ],
- "properties": {
- "iid": {
- "type": "number",
- "description": "ID that is unique in the scope of a parent object, and specific to the resource type."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- }
- },
- "remediations": {
- "type": "array",
- "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
- "items": {
- "type": "object",
- "required": [
- "fixes",
- "summary",
- "diff"
- ],
- "properties": {
- "fixes": {
- "type": "array",
- "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
- "items": {
- "type": "object",
- "required": [
- "cve"
- ],
- "properties": {
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- }
- }
- }
- },
- "summary": {
- "type": "string",
- "minLength": 1,
- "description": "An overview of how the vulnerabilities were fixed."
- },
- "diff": {
- "type": "string",
- "minLength": 1,
- "description": "A base64-encoded remediation code diff, compatible with git apply."
- }
- }
- }
- },
- "dependency_files": {
- "type": "array",
- "description": "List of dependency files identified in the project.",
- "items": {
- "type": "object",
- "required": [
- "path",
- "package_manager",
- "dependencies"
- ],
- "properties": {
- "path": {
- "type": "string",
- "minLength": 1
- },
- "package_manager": {
- "type": "string",
- "minLength": 1
- },
- "dependencies": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Describes the dependency of a project where the vulnerability is located.",
- "properties": {
- "package": {
- "type": "object",
- "description": "Provides information on the package where the vulnerability is located.",
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the package where the vulnerability is located."
- }
- }
- },
- "version": {
- "type": "string",
- "description": "Version of the vulnerable package."
- },
- "iid": {
- "description": "ID that identifies the dependency in the scope of a dependency file.",
- "type": "number"
- },
- "direct": {
- "type": "boolean",
- "description": "Tells whether this is a direct, top-level dependency of the scanned project."
- },
- "dependency_path": {
- "type": "array",
- "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.",
- "items": {
- "type": "object",
- "required": [
- "iid"
- ],
- "properties": {
- "iid": {
- "type": "number",
- "description": "ID that is unique in the scope of a parent object, and specific to the resource type."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- }
- }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.1/sast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.1/sast-report-format.json
deleted file mode 100644
index 7c2cd2b78cf..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.1/sast-report-format.json
+++ /dev/null
@@ -1,774 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Report format for GitLab SAST",
- "description": "This schema provides the report format for Static Application Security Testing analyzers (https://docs.gitlab.com/ee/user/application_security/sast).",
- "definitions": {
- "detail_type": {
- "oneOf": [
- {
- "$ref": "#/definitions/named_list"
- },
- {
- "$ref": "#/definitions/list"
- },
- {
- "$ref": "#/definitions/table"
- },
- {
- "$ref": "#/definitions/text"
- },
- {
- "$ref": "#/definitions/url"
- },
- {
- "$ref": "#/definitions/code"
- },
- {
- "$ref": "#/definitions/value"
- },
- {
- "$ref": "#/definitions/diff"
- },
- {
- "$ref": "#/definitions/markdown"
- },
- {
- "$ref": "#/definitions/commit"
- },
- {
- "$ref": "#/definitions/file_location"
- },
- {
- "$ref": "#/definitions/module_location"
- }
- ]
- },
- "text_value": {
- "type": "string"
- },
- "named_field": {
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "$ref": "#/definitions/text_value",
- "minLength": 1
- },
- "description": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "named_list": {
- "type": "object",
- "description": "An object with named and typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "named-list"
- },
- "items": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- {
- "$ref": "#/definitions/named_field"
- },
- {
- "$ref": "#/definitions/detail_type"
- }
- ]
- }
- }
- }
- }
- },
- "list": {
- "type": "object",
- "description": "A list of typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- },
- "table": {
- "type": "object",
- "description": "A table of typed fields",
- "required": [
- "type",
- "rows"
- ],
- "properties": {
- "type": {
- "const": "table"
- },
- "header": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- },
- "rows": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- }
- },
- "text": {
- "type": "object",
- "description": "Raw text",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "text"
- },
- "value": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "url": {
- "type": "object",
- "description": "A single URL",
- "required": [
- "type",
- "href"
- ],
- "properties": {
- "type": {
- "const": "url"
- },
- "text": {
- "$ref": "#/definitions/text_value"
- },
- "href": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "http://mysite.com"
- ]
- }
- }
- },
- "code": {
- "type": "object",
- "description": "A codeblock",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "code"
- },
- "value": {
- "type": "string"
- },
- "lang": {
- "type": "string",
- "description": "A programming language"
- }
- }
- },
- "value": {
- "type": "object",
- "description": "A field that can store a range of types of value",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "value"
- },
- "value": {
- "type": [
- "number",
- "string",
- "boolean"
- ]
- }
- }
- },
- "diff": {
- "type": "object",
- "description": "A diff",
- "required": [
- "type",
- "before",
- "after"
- ],
- "properties": {
- "type": {
- "const": "diff"
- },
- "before": {
- "type": "string"
- },
- "after": {
- "type": "string"
- }
- }
- },
- "markdown": {
- "type": "object",
- "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "markdown"
- },
- "value": {
- "$ref": "#/definitions/text_value",
- "examples": [
- "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
- ]
- }
- }
- },
- "commit": {
- "type": "object",
- "description": "A commit/tag/branch within the GitLab project",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "commit"
- },
- "value": {
- "type": "string",
- "description": "The commit SHA",
- "minLength": 1
- }
- }
- },
- "file_location": {
- "type": "object",
- "description": "A location within a file in the project",
- "required": [
- "type",
- "file_name",
- "line_start"
- ],
- "properties": {
- "type": {
- "const": "file-location"
- },
- "file_name": {
- "type": "string",
- "minLength": 1
- },
- "line_start": {
- "type": "integer"
- },
- "line_end": {
- "type": "integer"
- }
- }
- },
- "module_location": {
- "type": "object",
- "description": "A location within a binary module of the form module+relative_offset",
- "required": [
- "type",
- "module_name",
- "offset"
- ],
- "properties": {
- "type": {
- "const": "module-location"
- },
- "module_name": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "compiled_binary"
- ]
- },
- "offset": {
- "type": "integer",
- "examples": [
- 100
- ]
- }
- }
- }
- },
- "self": {
- "version": "14.0.1"
- },
- "required": [
- "version",
- "vulnerabilities"
- ],
- "additionalProperties": true,
- "properties": {
- "scan": {
- "type": "object",
- "required": [
- "end_time",
- "scanner",
- "start_time",
- "status",
- "type"
- ],
- "properties": {
- "end_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-01-28T03:26:02"
- ]
- },
- "messages": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Communication intended for the initiator of a scan.",
- "required": [
- "level",
- "value"
- ],
- "properties": {
- "level": {
- "type": "string",
- "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
- "enum": [
- "info",
- "warn",
- "fatal"
- ],
- "examples": [
- "info"
- ]
- },
- "value": {
- "type": "string",
- "description": "The message to communicate.",
- "minLength": 1,
- "examples": [
- "Permission denied, scanning aborted"
- ]
- }
- }
- }
- },
- "scanner": {
- "type": "object",
- "description": "Object defining the scanner used to perform the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the scanner.",
- "minLength": 1,
- "examples": [
- "my-sast-scanner"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the scanner, not required to be unique.",
- "minLength": 1,
- "examples": [
- "My SAST Scanner"
- ]
- },
- "url": {
- "type": "string",
- "description": "A link to more information about the scanner.",
- "examples": [
- "https://scanner.url"
- ]
- },
- "version": {
- "type": "string",
- "description": "The version of the scanner.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- },
- "vendor": {
- "type": "object",
- "description": "The vendor/maintainer of the scanner.",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- }
- }
- },
- "start_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-02-14T16:01:59"
- ]
- },
- "status": {
- "type": "string",
- "description": "Result of the scan.",
- "enum": [
- "success",
- "failure"
- ]
- },
- "type": {
- "type": "string",
- "description": "Type of the scan.",
- "enum": [
- "sast"
- ]
- }
- }
- },
- "schema": {
- "type": "string",
- "description": "URI pointing to the validating security report schema.",
- "format": "uri"
- },
- "version": {
- "type": "string",
- "description": "The version of the schema to which the JSON report conforms.",
- "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
- },
- "vulnerabilities": {
- "type": "array",
- "description": "Array of vulnerability objects.",
- "items": {
- "type": "object",
- "description": "Describes the vulnerability.",
- "required": [
- "category",
- "cve",
- "identifiers",
- "location",
- "scanner"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
- "examples": [
- "642735a5-1425-428d-8d4e-3c854885a3c9"
- ]
- },
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
- "name": {
- "type": "string",
- "description": "The name of the vulnerability. This must not include the finding's specific information."
- },
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
- "description": {
- "type": "string",
- "description": "A long text section describing the vulnerability more fully."
- },
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
- "severity": {
- "type": "string",
- "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Info",
- "Unknown",
- "Low",
- "Medium",
- "High",
- "Critical"
- ]
- },
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
- "solution": {
- "type": "string",
- "description": "Explanation of how to fix the vulnerability."
- },
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
- "identifiers": {
- "type": "array",
- "minItems": 1,
- "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
- "items": {
- "type": "object",
- "required": [
- "type",
- "name",
- "value"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
- "minLength": 1
- },
- "name": {
- "type": "string",
- "description": "Human-readable name of the identifier.",
- "minLength": 1
- },
- "url": {
- "type": "string",
- "description": "URL of the identifier's documentation.",
- "format": "uri"
- },
- "value": {
- "type": "string",
- "description": "Value of the identifier, for matching purpose.",
- "minLength": 1
- }
- }
- }
- },
- "links": {
- "type": "array",
- "description": "An array of references to external documentation or articles that describe the vulnerability.",
- "items": {
- "type": "object",
- "required": [
- "url"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the vulnerability details link."
- },
- "url": {
- "type": "string",
- "description": "URL of the vulnerability details document.",
- "format": "uri"
- }
- }
- }
- },
- "details": {
- "$ref": "#/definitions/named_list/properties/items"
- },
- "tracking": {
- "description": "Describes how this vulnerability should be tracked as the project changes.",
- "oneOf": [
- {
- "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
- "required": [
- "items"
- ],
- "properties": {
- "type": {
- "const": "source"
- },
- "items": {
- "type": "array",
- "items": {
- "description": "An item that should be tracked using source-specific tracking methods.",
- "type": "object",
- "required": [
- "signatures"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the file that includes the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the file that includes the vulnerability."
- },
- "signatures": {
- "type": "array",
- "description": "An array of calculated tracking signatures for this tracking item.",
- "minItems": 1,
- "items": {
- "description": "A calculated tracking signature value and metadata.",
- "required": [
- "algorithm",
- "value"
- ],
- "properties": {
- "algorithm": {
- "type": "string",
- "description": "The algorithm used to generate the signature."
- },
- "value": {
- "type": "string",
- "description": "The result of this signature algorithm."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "Each tracking type must declare its own type."
- }
- }
- },
- "location": {
- "type": "object",
- "description": "Identifies the vulnerability's location.",
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the code affected by the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the code affected by the vulnerability."
- },
- "class": {
- "type": "string",
- "description": "Provides the name of the class where the vulnerability is located."
- },
- "method": {
- "type": "string",
- "description": "Provides the name of the method where the vulnerability is located."
- }
- }
- },
- "raw_source_code_extract": {
- "type": "string",
- "description": "Provides an unsanitized excerpt of the affected source code."
- }
- }
- }
- },
- "remediations": {
- "type": "array",
- "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
- "items": {
- "type": "object",
- "required": [
- "fixes",
- "summary",
- "diff"
- ],
- "properties": {
- "fixes": {
- "type": "array",
- "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
- "items": {
- "type": "object",
- "required": [
- "cve"
- ],
- "properties": {
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- }
- }
- }
- },
- "summary": {
- "type": "string",
- "minLength": 1,
- "description": "An overview of how the vulnerabilities were fixed."
- },
- "diff": {
- "type": "string",
- "minLength": 1,
- "description": "A base64-encoded remediation code diff, compatible with git apply."
- }
- }
- }
- }
- }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.1/secret-detection-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.1/secret-detection-report-format.json
deleted file mode 100644
index b4449d0d59c..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.1/secret-detection-report-format.json
+++ /dev/null
@@ -1,797 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Report format for GitLab Secret Detection",
- "description": "This schema provides the the report format for the Secret Detection analyzer (https://docs.gitlab.com/ee/user/application_security/secret_detection)",
- "definitions": {
- "detail_type": {
- "oneOf": [
- {
- "$ref": "#/definitions/named_list"
- },
- {
- "$ref": "#/definitions/list"
- },
- {
- "$ref": "#/definitions/table"
- },
- {
- "$ref": "#/definitions/text"
- },
- {
- "$ref": "#/definitions/url"
- },
- {
- "$ref": "#/definitions/code"
- },
- {
- "$ref": "#/definitions/value"
- },
- {
- "$ref": "#/definitions/diff"
- },
- {
- "$ref": "#/definitions/markdown"
- },
- {
- "$ref": "#/definitions/commit"
- },
- {
- "$ref": "#/definitions/file_location"
- },
- {
- "$ref": "#/definitions/module_location"
- }
- ]
- },
- "text_value": {
- "type": "string"
- },
- "named_field": {
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "$ref": "#/definitions/text_value",
- "minLength": 1
- },
- "description": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "named_list": {
- "type": "object",
- "description": "An object with named and typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "named-list"
- },
- "items": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- {
- "$ref": "#/definitions/named_field"
- },
- {
- "$ref": "#/definitions/detail_type"
- }
- ]
- }
- }
- }
- }
- },
- "list": {
- "type": "object",
- "description": "A list of typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- },
- "table": {
- "type": "object",
- "description": "A table of typed fields",
- "required": [
- "type",
- "rows"
- ],
- "properties": {
- "type": {
- "const": "table"
- },
- "header": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- },
- "rows": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- }
- },
- "text": {
- "type": "object",
- "description": "Raw text",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "text"
- },
- "value": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "url": {
- "type": "object",
- "description": "A single URL",
- "required": [
- "type",
- "href"
- ],
- "properties": {
- "type": {
- "const": "url"
- },
- "text": {
- "$ref": "#/definitions/text_value"
- },
- "href": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "http://mysite.com"
- ]
- }
- }
- },
- "code": {
- "type": "object",
- "description": "A codeblock",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "code"
- },
- "value": {
- "type": "string"
- },
- "lang": {
- "type": "string",
- "description": "A programming language"
- }
- }
- },
- "value": {
- "type": "object",
- "description": "A field that can store a range of types of value",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "value"
- },
- "value": {
- "type": [
- "number",
- "string",
- "boolean"
- ]
- }
- }
- },
- "diff": {
- "type": "object",
- "description": "A diff",
- "required": [
- "type",
- "before",
- "after"
- ],
- "properties": {
- "type": {
- "const": "diff"
- },
- "before": {
- "type": "string"
- },
- "after": {
- "type": "string"
- }
- }
- },
- "markdown": {
- "type": "object",
- "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "markdown"
- },
- "value": {
- "$ref": "#/definitions/text_value",
- "examples": [
- "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
- ]
- }
- }
- },
- "commit": {
- "type": "object",
- "description": "A commit/tag/branch within the GitLab project",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "commit"
- },
- "value": {
- "type": "string",
- "description": "The commit SHA",
- "minLength": 1
- }
- }
- },
- "file_location": {
- "type": "object",
- "description": "A location within a file in the project",
- "required": [
- "type",
- "file_name",
- "line_start"
- ],
- "properties": {
- "type": {
- "const": "file-location"
- },
- "file_name": {
- "type": "string",
- "minLength": 1
- },
- "line_start": {
- "type": "integer"
- },
- "line_end": {
- "type": "integer"
- }
- }
- },
- "module_location": {
- "type": "object",
- "description": "A location within a binary module of the form module+relative_offset",
- "required": [
- "type",
- "module_name",
- "offset"
- ],
- "properties": {
- "type": {
- "const": "module-location"
- },
- "module_name": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "compiled_binary"
- ]
- },
- "offset": {
- "type": "integer",
- "examples": [
- 100
- ]
- }
- }
- }
- },
- "self": {
- "version": "14.0.1"
- },
- "required": [
- "version",
- "vulnerabilities"
- ],
- "additionalProperties": true,
- "properties": {
- "scan": {
- "type": "object",
- "required": [
- "end_time",
- "scanner",
- "start_time",
- "status",
- "type"
- ],
- "properties": {
- "end_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-01-28T03:26:02"
- ]
- },
- "messages": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Communication intended for the initiator of a scan.",
- "required": [
- "level",
- "value"
- ],
- "properties": {
- "level": {
- "type": "string",
- "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
- "enum": [
- "info",
- "warn",
- "fatal"
- ],
- "examples": [
- "info"
- ]
- },
- "value": {
- "type": "string",
- "description": "The message to communicate.",
- "minLength": 1,
- "examples": [
- "Permission denied, scanning aborted"
- ]
- }
- }
- }
- },
- "scanner": {
- "type": "object",
- "description": "Object defining the scanner used to perform the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the scanner.",
- "minLength": 1,
- "examples": [
- "my-sast-scanner"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the scanner, not required to be unique.",
- "minLength": 1,
- "examples": [
- "My SAST Scanner"
- ]
- },
- "url": {
- "type": "string",
- "description": "A link to more information about the scanner.",
- "examples": [
- "https://scanner.url"
- ]
- },
- "version": {
- "type": "string",
- "description": "The version of the scanner.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- },
- "vendor": {
- "type": "object",
- "description": "The vendor/maintainer of the scanner.",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- }
- }
- },
- "start_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-02-14T16:01:59"
- ]
- },
- "status": {
- "type": "string",
- "description": "Result of the scan.",
- "enum": [
- "success",
- "failure"
- ]
- },
- "type": {
- "type": "string",
- "description": "Type of the scan.",
- "enum": [
- "secret_detection"
- ]
- }
- }
- },
- "schema": {
- "type": "string",
- "description": "URI pointing to the validating security report schema.",
- "format": "uri"
- },
- "version": {
- "type": "string",
- "description": "The version of the schema to which the JSON report conforms.",
- "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
- },
- "vulnerabilities": {
- "type": "array",
- "description": "Array of vulnerability objects.",
- "items": {
- "type": "object",
- "description": "Describes the vulnerability.",
- "required": [
- "category",
- "cve",
- "identifiers",
- "location",
- "scanner"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
- "examples": [
- "642735a5-1425-428d-8d4e-3c854885a3c9"
- ]
- },
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
- "name": {
- "type": "string",
- "description": "The name of the vulnerability. This must not include the finding's specific information."
- },
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
- "description": {
- "type": "string",
- "description": "A long text section describing the vulnerability more fully."
- },
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
- "severity": {
- "type": "string",
- "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Info",
- "Unknown",
- "Low",
- "Medium",
- "High",
- "Critical"
- ]
- },
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
- "solution": {
- "type": "string",
- "description": "Explanation of how to fix the vulnerability."
- },
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
- "identifiers": {
- "type": "array",
- "minItems": 1,
- "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
- "items": {
- "type": "object",
- "required": [
- "type",
- "name",
- "value"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
- "minLength": 1
- },
- "name": {
- "type": "string",
- "description": "Human-readable name of the identifier.",
- "minLength": 1
- },
- "url": {
- "type": "string",
- "description": "URL of the identifier's documentation.",
- "format": "uri"
- },
- "value": {
- "type": "string",
- "description": "Value of the identifier, for matching purpose.",
- "minLength": 1
- }
- }
- }
- },
- "links": {
- "type": "array",
- "description": "An array of references to external documentation or articles that describe the vulnerability.",
- "items": {
- "type": "object",
- "required": [
- "url"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the vulnerability details link."
- },
- "url": {
- "type": "string",
- "description": "URL of the vulnerability details document.",
- "format": "uri"
- }
- }
- }
- },
- "details": {
- "$ref": "#/definitions/named_list/properties/items"
- },
- "tracking": {
- "description": "Describes how this vulnerability should be tracked as the project changes.",
- "oneOf": [
- {
- "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
- "required": [
- "items"
- ],
- "properties": {
- "type": {
- "const": "source"
- },
- "items": {
- "type": "array",
- "items": {
- "description": "An item that should be tracked using source-specific tracking methods.",
- "type": "object",
- "required": [
- "signatures"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the file that includes the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the file that includes the vulnerability."
- },
- "signatures": {
- "type": "array",
- "description": "An array of calculated tracking signatures for this tracking item.",
- "minItems": 1,
- "items": {
- "description": "A calculated tracking signature value and metadata.",
- "required": [
- "algorithm",
- "value"
- ],
- "properties": {
- "algorithm": {
- "type": "string",
- "description": "The algorithm used to generate the signature."
- },
- "value": {
- "type": "string",
- "description": "The result of this signature algorithm."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "Each tracking type must declare its own type."
- }
- }
- },
- "location": {
- "required": [
- "commit"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located"
- },
- "commit": {
- "type": "object",
- "description": "Represents the commit in which the vulnerability was detected",
- "required": [
- "sha"
- ],
- "properties": {
- "author": {
- "type": "string"
- },
- "date": {
- "type": "string"
- },
- "message": {
- "type": "string"
- },
- "sha": {
- "type": "string",
- "minLength": 1
- }
- }
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the code affected by the vulnerability"
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the code affected by the vulnerability"
- },
- "class": {
- "type": "string",
- "description": "Provides the name of the class where the vulnerability is located"
- },
- "method": {
- "type": "string",
- "description": "Provides the name of the method where the vulnerability is located"
- }
- }
- },
- "raw_source_code_extract": {
- "type": "string",
- "description": "Provides an unsanitized excerpt of the affected source code."
- }
- }
- }
- },
- "remediations": {
- "type": "array",
- "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
- "items": {
- "type": "object",
- "required": [
- "fixes",
- "summary",
- "diff"
- ],
- "properties": {
- "fixes": {
- "type": "array",
- "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
- "items": {
- "type": "object",
- "required": [
- "cve"
- ],
- "properties": {
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- }
- }
- }
- },
- "summary": {
- "type": "string",
- "minLength": 1,
- "description": "An overview of how the vulnerabilities were fixed."
- },
- "diff": {
- "type": "string",
- "minLength": 1,
- "description": "A base64-encoded remediation code diff, compatible with git apply."
- }
- }
- }
- }
- }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.2/container-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.2/container-scanning-report-format.json
deleted file mode 100644
index 696fa214abd..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.2/container-scanning-report-format.json
+++ /dev/null
@@ -1,871 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Report format for GitLab Container Scanning",
- "description": "This schema provides the the report format for Container Scanning (https://docs.gitlab.com/ee/user/application_security/container_scanning).",
- "definitions": {
- "detail_type": {
- "oneOf": [
- {
- "$ref": "#/definitions/named_list"
- },
- {
- "$ref": "#/definitions/list"
- },
- {
- "$ref": "#/definitions/table"
- },
- {
- "$ref": "#/definitions/text"
- },
- {
- "$ref": "#/definitions/url"
- },
- {
- "$ref": "#/definitions/code"
- },
- {
- "$ref": "#/definitions/value"
- },
- {
- "$ref": "#/definitions/diff"
- },
- {
- "$ref": "#/definitions/markdown"
- },
- {
- "$ref": "#/definitions/commit"
- },
- {
- "$ref": "#/definitions/file_location"
- },
- {
- "$ref": "#/definitions/module_location"
- }
- ]
- },
- "text_value": {
- "type": "string"
- },
- "named_field": {
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "$ref": "#/definitions/text_value",
- "minLength": 1
- },
- "description": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "named_list": {
- "type": "object",
- "description": "An object with named and typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "named-list"
- },
- "items": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- {
- "$ref": "#/definitions/named_field"
- },
- {
- "$ref": "#/definitions/detail_type"
- }
- ]
- }
- }
- }
- }
- },
- "list": {
- "type": "object",
- "description": "A list of typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- },
- "table": {
- "type": "object",
- "description": "A table of typed fields",
- "required": [
- "type",
- "rows"
- ],
- "properties": {
- "type": {
- "const": "table"
- },
- "header": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- },
- "rows": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- }
- },
- "text": {
- "type": "object",
- "description": "Raw text",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "text"
- },
- "value": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "url": {
- "type": "object",
- "description": "A single URL",
- "required": [
- "type",
- "href"
- ],
- "properties": {
- "type": {
- "const": "url"
- },
- "text": {
- "$ref": "#/definitions/text_value"
- },
- "href": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "http://mysite.com"
- ]
- }
- }
- },
- "code": {
- "type": "object",
- "description": "A codeblock",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "code"
- },
- "value": {
- "type": "string"
- },
- "lang": {
- "type": "string",
- "description": "A programming language"
- }
- }
- },
- "value": {
- "type": "object",
- "description": "A field that can store a range of types of value",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "value"
- },
- "value": {
- "type": [
- "number",
- "string",
- "boolean"
- ]
- }
- }
- },
- "diff": {
- "type": "object",
- "description": "A diff",
- "required": [
- "type",
- "before",
- "after"
- ],
- "properties": {
- "type": {
- "const": "diff"
- },
- "before": {
- "type": "string"
- },
- "after": {
- "type": "string"
- }
- }
- },
- "markdown": {
- "type": "object",
- "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "markdown"
- },
- "value": {
- "$ref": "#/definitions/text_value",
- "examples": [
- "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
- ]
- }
- }
- },
- "commit": {
- "type": "object",
- "description": "A commit/tag/branch within the GitLab project",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "commit"
- },
- "value": {
- "type": "string",
- "description": "The commit SHA",
- "minLength": 1
- }
- }
- },
- "file_location": {
- "type": "object",
- "description": "A location within a file in the project",
- "required": [
- "type",
- "file_name",
- "line_start"
- ],
- "properties": {
- "type": {
- "const": "file-location"
- },
- "file_name": {
- "type": "string",
- "minLength": 1
- },
- "line_start": {
- "type": "integer"
- },
- "line_end": {
- "type": "integer"
- }
- }
- },
- "module_location": {
- "type": "object",
- "description": "A location within a binary module of the form module+relative_offset",
- "required": [
- "type",
- "module_name",
- "offset"
- ],
- "properties": {
- "type": {
- "const": "module-location"
- },
- "module_name": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "compiled_binary"
- ]
- },
- "offset": {
- "type": "integer",
- "examples": [
- 100
- ]
- }
- }
- }
- },
- "self": {
- "version": "14.0.2"
- },
- "required": [
- "version",
- "vulnerabilities"
- ],
- "additionalProperties": true,
- "properties": {
- "scan": {
- "type": "object",
- "required": [
- "end_time",
- "scanner",
- "start_time",
- "status",
- "type"
- ],
- "properties": {
- "end_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-01-28T03:26:02"
- ]
- },
- "messages": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Communication intended for the initiator of a scan.",
- "required": [
- "level",
- "value"
- ],
- "properties": {
- "level": {
- "type": "string",
- "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
- "enum": [
- "info",
- "warn",
- "fatal"
- ],
- "examples": [
- "info"
- ]
- },
- "value": {
- "type": "string",
- "description": "The message to communicate.",
- "minLength": 1,
- "examples": [
- "Permission denied, scanning aborted"
- ]
- }
- }
- }
- },
- "analyzer": {
- "type": "object",
- "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the analyzer.",
- "minLength": 1,
- "examples": [
- "gitlab-dast"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the analyzer, not required to be unique.",
- "minLength": 1,
- "examples": [
- "GitLab DAST"
- ]
- },
- "url": {
- "type": "string",
- "format": "uri",
- "pattern": "^https?://.+",
- "description": "A link to more information about the analyzer.",
- "examples": [
- "https://docs.gitlab.com/ee/user/application_security/dast"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the analyzer.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- },
- "version": {
- "type": "string",
- "description": "The version of the analyzer.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- }
- }
- },
- "scanner": {
- "type": "object",
- "description": "Object defining the scanner used to perform the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the scanner.",
- "minLength": 1,
- "examples": [
- "my-sast-scanner"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the scanner, not required to be unique.",
- "minLength": 1,
- "examples": [
- "My SAST Scanner"
- ]
- },
- "url": {
- "type": "string",
- "description": "A link to more information about the scanner.",
- "examples": [
- "https://scanner.url"
- ]
- },
- "version": {
- "type": "string",
- "description": "The version of the scanner.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the scanner.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- }
- }
- },
- "start_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-02-14T16:01:59"
- ]
- },
- "status": {
- "type": "string",
- "description": "Result of the scan.",
- "enum": [
- "success",
- "failure"
- ]
- },
- "type": {
- "type": "string",
- "description": "Type of the scan.",
- "enum": [
- "container_scanning"
- ]
- }
- }
- },
- "schema": {
- "type": "string",
- "description": "URI pointing to the validating security report schema.",
- "format": "uri"
- },
- "version": {
- "type": "string",
- "description": "The version of the schema to which the JSON report conforms.",
- "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
- },
- "vulnerabilities": {
- "type": "array",
- "description": "Array of vulnerability objects.",
- "items": {
- "type": "object",
- "description": "Describes the vulnerability.",
- "required": [
- "category",
- "cve",
- "identifiers",
- "location",
- "scanner"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
- "examples": [
- "642735a5-1425-428d-8d4e-3c854885a3c9"
- ]
- },
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
- "name": {
- "type": "string",
- "description": "The name of the vulnerability. This must not include the finding's specific information."
- },
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
- "description": {
- "type": "string",
- "description": "A long text section describing the vulnerability more fully."
- },
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
- "severity": {
- "type": "string",
- "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Info",
- "Unknown",
- "Low",
- "Medium",
- "High",
- "Critical"
- ]
- },
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
- "solution": {
- "type": "string",
- "description": "Explanation of how to fix the vulnerability."
- },
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
- "identifiers": {
- "type": "array",
- "minItems": 1,
- "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
- "items": {
- "type": "object",
- "required": [
- "type",
- "name",
- "value"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
- "minLength": 1
- },
- "name": {
- "type": "string",
- "description": "Human-readable name of the identifier.",
- "minLength": 1
- },
- "url": {
- "type": "string",
- "description": "URL of the identifier's documentation.",
- "format": "uri"
- },
- "value": {
- "type": "string",
- "description": "Value of the identifier, for matching purpose.",
- "minLength": 1
- }
- }
- }
- },
- "links": {
- "type": "array",
- "description": "An array of references to external documentation or articles that describe the vulnerability.",
- "items": {
- "type": "object",
- "required": [
- "url"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the vulnerability details link."
- },
- "url": {
- "type": "string",
- "description": "URL of the vulnerability details document.",
- "format": "uri"
- }
- }
- }
- },
- "details": {
- "$ref": "#/definitions/named_list/properties/items"
- },
- "tracking": {
- "description": "Describes how this vulnerability should be tracked as the project changes.",
- "oneOf": [
- {
- "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
- "required": [
- "items"
- ],
- "properties": {
- "type": {
- "const": "source"
- },
- "items": {
- "type": "array",
- "items": {
- "description": "An item that should be tracked using source-specific tracking methods.",
- "type": "object",
- "required": [
- "signatures"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the file that includes the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the file that includes the vulnerability."
- },
- "signatures": {
- "type": "array",
- "description": "An array of calculated tracking signatures for this tracking item.",
- "minItems": 1,
- "items": {
- "description": "A calculated tracking signature value and metadata.",
- "required": [
- "algorithm",
- "value"
- ],
- "properties": {
- "algorithm": {
- "type": "string",
- "description": "The algorithm used to generate the signature."
- },
- "value": {
- "type": "string",
- "description": "The result of this signature algorithm."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "Each tracking type must declare its own type."
- }
- }
- },
- "location": {
- "type": "object",
- "description": "Identifies the vulnerability's location.",
- "required": [
- "dependency",
- "operating_system",
- "image"
- ],
- "properties": {
- "dependency": {
- "type": "object",
- "description": "Describes the dependency of a project where the vulnerability is located.",
- "properties": {
- "package": {
- "type": "object",
- "description": "Provides information on the package where the vulnerability is located.",
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the package where the vulnerability is located."
- }
- }
- },
- "version": {
- "type": "string",
- "description": "Version of the vulnerable package."
- },
- "iid": {
- "description": "ID that identifies the dependency in the scope of a dependency file.",
- "type": "number"
- },
- "direct": {
- "type": "boolean",
- "description": "Tells whether this is a direct, top-level dependency of the scanned project."
- },
- "dependency_path": {
- "type": "array",
- "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.",
- "items": {
- "type": "object",
- "required": [
- "iid"
- ],
- "properties": {
- "iid": {
- "type": "number",
- "description": "ID that is unique in the scope of a parent object, and specific to the resource type."
- }
- }
- }
- }
- }
- },
- "operating_system": {
- "type": "string",
- "minLength": 1,
- "description": "The operating system that contains the vulnerable package."
- },
- "image": {
- "type": "string",
- "minLength": 1,
- "description": "The analyzed Docker image."
- }
- }
- }
- }
- }
- },
- "remediations": {
- "type": "array",
- "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
- "items": {
- "type": "object",
- "required": [
- "fixes",
- "summary",
- "diff"
- ],
- "properties": {
- "fixes": {
- "type": "array",
- "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
- "items": {
- "type": "object",
- "required": [
- "cve"
- ],
- "properties": {
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- }
- }
- }
- },
- "summary": {
- "type": "string",
- "minLength": 1,
- "description": "An overview of how the vulnerabilities were fixed."
- },
- "diff": {
- "type": "string",
- "minLength": 1,
- "description": "A base64-encoded remediation code diff, compatible with git apply."
- }
- }
- }
- }
- }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.2/coverage-fuzzing-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.2/coverage-fuzzing-report-format.json
deleted file mode 100644
index 1312696d642..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.2/coverage-fuzzing-report-format.json
+++ /dev/null
@@ -1,841 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Report format for GitLab Fuzz Testing",
- "description": "This schema provides the report format for Coverage Guided Fuzz Testing (https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing).",
- "definitions": {
- "detail_type": {
- "oneOf": [
- {
- "$ref": "#/definitions/named_list"
- },
- {
- "$ref": "#/definitions/list"
- },
- {
- "$ref": "#/definitions/table"
- },
- {
- "$ref": "#/definitions/text"
- },
- {
- "$ref": "#/definitions/url"
- },
- {
- "$ref": "#/definitions/code"
- },
- {
- "$ref": "#/definitions/value"
- },
- {
- "$ref": "#/definitions/diff"
- },
- {
- "$ref": "#/definitions/markdown"
- },
- {
- "$ref": "#/definitions/commit"
- },
- {
- "$ref": "#/definitions/file_location"
- },
- {
- "$ref": "#/definitions/module_location"
- }
- ]
- },
- "text_value": {
- "type": "string"
- },
- "named_field": {
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "$ref": "#/definitions/text_value",
- "minLength": 1
- },
- "description": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "named_list": {
- "type": "object",
- "description": "An object with named and typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "named-list"
- },
- "items": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- {
- "$ref": "#/definitions/named_field"
- },
- {
- "$ref": "#/definitions/detail_type"
- }
- ]
- }
- }
- }
- }
- },
- "list": {
- "type": "object",
- "description": "A list of typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- },
- "table": {
- "type": "object",
- "description": "A table of typed fields",
- "required": [
- "type",
- "rows"
- ],
- "properties": {
- "type": {
- "const": "table"
- },
- "header": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- },
- "rows": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- }
- },
- "text": {
- "type": "object",
- "description": "Raw text",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "text"
- },
- "value": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "url": {
- "type": "object",
- "description": "A single URL",
- "required": [
- "type",
- "href"
- ],
- "properties": {
- "type": {
- "const": "url"
- },
- "text": {
- "$ref": "#/definitions/text_value"
- },
- "href": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "http://mysite.com"
- ]
- }
- }
- },
- "code": {
- "type": "object",
- "description": "A codeblock",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "code"
- },
- "value": {
- "type": "string"
- },
- "lang": {
- "type": "string",
- "description": "A programming language"
- }
- }
- },
- "value": {
- "type": "object",
- "description": "A field that can store a range of types of value",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "value"
- },
- "value": {
- "type": [
- "number",
- "string",
- "boolean"
- ]
- }
- }
- },
- "diff": {
- "type": "object",
- "description": "A diff",
- "required": [
- "type",
- "before",
- "after"
- ],
- "properties": {
- "type": {
- "const": "diff"
- },
- "before": {
- "type": "string"
- },
- "after": {
- "type": "string"
- }
- }
- },
- "markdown": {
- "type": "object",
- "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "markdown"
- },
- "value": {
- "$ref": "#/definitions/text_value",
- "examples": [
- "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
- ]
- }
- }
- },
- "commit": {
- "type": "object",
- "description": "A commit/tag/branch within the GitLab project",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "commit"
- },
- "value": {
- "type": "string",
- "description": "The commit SHA",
- "minLength": 1
- }
- }
- },
- "file_location": {
- "type": "object",
- "description": "A location within a file in the project",
- "required": [
- "type",
- "file_name",
- "line_start"
- ],
- "properties": {
- "type": {
- "const": "file-location"
- },
- "file_name": {
- "type": "string",
- "minLength": 1
- },
- "line_start": {
- "type": "integer"
- },
- "line_end": {
- "type": "integer"
- }
- }
- },
- "module_location": {
- "type": "object",
- "description": "A location within a binary module of the form module+relative_offset",
- "required": [
- "type",
- "module_name",
- "offset"
- ],
- "properties": {
- "type": {
- "const": "module-location"
- },
- "module_name": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "compiled_binary"
- ]
- },
- "offset": {
- "type": "integer",
- "examples": [
- 100
- ]
- }
- }
- }
- },
- "self": {
- "version": "14.0.2"
- },
- "required": [
- "version",
- "vulnerabilities"
- ],
- "additionalProperties": true,
- "properties": {
- "scan": {
- "type": "object",
- "required": [
- "end_time",
- "scanner",
- "start_time",
- "status",
- "type"
- ],
- "properties": {
- "end_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-01-28T03:26:02"
- ]
- },
- "messages": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Communication intended for the initiator of a scan.",
- "required": [
- "level",
- "value"
- ],
- "properties": {
- "level": {
- "type": "string",
- "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
- "enum": [
- "info",
- "warn",
- "fatal"
- ],
- "examples": [
- "info"
- ]
- },
- "value": {
- "type": "string",
- "description": "The message to communicate.",
- "minLength": 1,
- "examples": [
- "Permission denied, scanning aborted"
- ]
- }
- }
- }
- },
- "analyzer": {
- "type": "object",
- "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the analyzer.",
- "minLength": 1,
- "examples": [
- "gitlab-dast"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the analyzer, not required to be unique.",
- "minLength": 1,
- "examples": [
- "GitLab DAST"
- ]
- },
- "url": {
- "type": "string",
- "format": "uri",
- "pattern": "^https?://.+",
- "description": "A link to more information about the analyzer.",
- "examples": [
- "https://docs.gitlab.com/ee/user/application_security/dast"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the analyzer.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- },
- "version": {
- "type": "string",
- "description": "The version of the analyzer.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- }
- }
- },
- "scanner": {
- "type": "object",
- "description": "Object defining the scanner used to perform the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the scanner.",
- "minLength": 1,
- "examples": [
- "my-sast-scanner"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the scanner, not required to be unique.",
- "minLength": 1,
- "examples": [
- "My SAST Scanner"
- ]
- },
- "url": {
- "type": "string",
- "description": "A link to more information about the scanner.",
- "examples": [
- "https://scanner.url"
- ]
- },
- "version": {
- "type": "string",
- "description": "The version of the scanner.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the scanner.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- }
- }
- },
- "start_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-02-14T16:01:59"
- ]
- },
- "status": {
- "type": "string",
- "description": "Result of the scan.",
- "enum": [
- "success",
- "failure"
- ]
- },
- "type": {
- "type": "string",
- "description": "Type of the scan.",
- "enum": [
- "coverage_fuzzing"
- ]
- }
- }
- },
- "schema": {
- "type": "string",
- "description": "URI pointing to the validating security report schema.",
- "format": "uri"
- },
- "version": {
- "type": "string",
- "description": "The version of the schema to which the JSON report conforms.",
- "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
- },
- "vulnerabilities": {
- "type": "array",
- "description": "Array of vulnerability objects.",
- "items": {
- "type": "object",
- "description": "Describes the vulnerability.",
- "required": [
- "category",
- "cve",
- "identifiers",
- "location",
- "scanner"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
- "examples": [
- "642735a5-1425-428d-8d4e-3c854885a3c9"
- ]
- },
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
- "name": {
- "type": "string",
- "description": "The name of the vulnerability. This must not include the finding's specific information."
- },
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
- "description": {
- "type": "string",
- "description": "A long text section describing the vulnerability more fully."
- },
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
- "severity": {
- "type": "string",
- "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Info",
- "Unknown",
- "Low",
- "Medium",
- "High",
- "Critical"
- ]
- },
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
- "solution": {
- "type": "string",
- "description": "Explanation of how to fix the vulnerability."
- },
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
- "identifiers": {
- "type": "array",
- "minItems": 1,
- "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
- "items": {
- "type": "object",
- "required": [
- "type",
- "name",
- "value"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
- "minLength": 1
- },
- "name": {
- "type": "string",
- "description": "Human-readable name of the identifier.",
- "minLength": 1
- },
- "url": {
- "type": "string",
- "description": "URL of the identifier's documentation.",
- "format": "uri"
- },
- "value": {
- "type": "string",
- "description": "Value of the identifier, for matching purpose.",
- "minLength": 1
- }
- }
- }
- },
- "links": {
- "type": "array",
- "description": "An array of references to external documentation or articles that describe the vulnerability.",
- "items": {
- "type": "object",
- "required": [
- "url"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the vulnerability details link."
- },
- "url": {
- "type": "string",
- "description": "URL of the vulnerability details document.",
- "format": "uri"
- }
- }
- }
- },
- "details": {
- "$ref": "#/definitions/named_list/properties/items"
- },
- "tracking": {
- "description": "Describes how this vulnerability should be tracked as the project changes.",
- "oneOf": [
- {
- "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
- "required": [
- "items"
- ],
- "properties": {
- "type": {
- "const": "source"
- },
- "items": {
- "type": "array",
- "items": {
- "description": "An item that should be tracked using source-specific tracking methods.",
- "type": "object",
- "required": [
- "signatures"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the file that includes the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the file that includes the vulnerability."
- },
- "signatures": {
- "type": "array",
- "description": "An array of calculated tracking signatures for this tracking item.",
- "minItems": 1,
- "items": {
- "description": "A calculated tracking signature value and metadata.",
- "required": [
- "algorithm",
- "value"
- ],
- "properties": {
- "algorithm": {
- "type": "string",
- "description": "The algorithm used to generate the signature."
- },
- "value": {
- "type": "string",
- "description": "The result of this signature algorithm."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "Each tracking type must declare its own type."
- }
- }
- },
- "location": {
- "description": "The location of the error",
- "type": "object",
- "properties": {
- "crash_address": {
- "type": "string",
- "description": "The relative address in memory were the crash occurred.",
- "examples": [
- "0xabababab"
- ]
- },
- "stacktrace_snippet": {
- "type": "string",
- "description": "The stack trace recorded during fuzzing resulting the crash.",
- "examples": [
- "func_a+0xabcd\nfunc_b+0xabcc"
- ]
- },
- "crash_state": {
- "type": "string",
- "description": "Minimised and normalized crash stack-trace (called crash_state).",
- "examples": [
- "func_a+0xa\nfunc_b+0xb\nfunc_c+0xc"
- ]
- },
- "crash_type": {
- "type": "string",
- "description": "Type of the crash.",
- "examples": [
- "Heap-Buffer-overflow",
- "Division-by-zero"
- ]
- }
- }
- }
- }
- }
- },
- "remediations": {
- "type": "array",
- "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
- "items": {
- "type": "object",
- "required": [
- "fixes",
- "summary",
- "diff"
- ],
- "properties": {
- "fixes": {
- "type": "array",
- "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
- "items": {
- "type": "object",
- "required": [
- "cve"
- ],
- "properties": {
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- }
- }
- }
- },
- "summary": {
- "type": "string",
- "minLength": 1,
- "description": "An overview of how the vulnerabilities were fixed."
- },
- "diff": {
- "type": "string",
- "minLength": 1,
- "description": "A base64-encoded remediation code diff, compatible with git apply."
- }
- }
- }
- }
- }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.2/dast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.2/dast-report-format.json
deleted file mode 100644
index a7e9f83e557..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.2/dast-report-format.json
+++ /dev/null
@@ -1,1258 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Report format for GitLab DAST",
- "description": "This schema provides the the report format for Dynamic Application Security Testing (https://docs.gitlab.com/ee/user/application_security/dast).",
- "definitions": {
- "detail_type": {
- "oneOf": [
- {
- "$ref": "#/definitions/named_list"
- },
- {
- "$ref": "#/definitions/list"
- },
- {
- "$ref": "#/definitions/table"
- },
- {
- "$ref": "#/definitions/text"
- },
- {
- "$ref": "#/definitions/url"
- },
- {
- "$ref": "#/definitions/code"
- },
- {
- "$ref": "#/definitions/value"
- },
- {
- "$ref": "#/definitions/diff"
- },
- {
- "$ref": "#/definitions/markdown"
- },
- {
- "$ref": "#/definitions/commit"
- },
- {
- "$ref": "#/definitions/file_location"
- },
- {
- "$ref": "#/definitions/module_location"
- }
- ]
- },
- "text_value": {
- "type": "string"
- },
- "named_field": {
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "$ref": "#/definitions/text_value",
- "minLength": 1
- },
- "description": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "named_list": {
- "type": "object",
- "description": "An object with named and typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "named-list"
- },
- "items": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- {
- "$ref": "#/definitions/named_field"
- },
- {
- "$ref": "#/definitions/detail_type"
- }
- ]
- }
- }
- }
- }
- },
- "list": {
- "type": "object",
- "description": "A list of typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- },
- "table": {
- "type": "object",
- "description": "A table of typed fields",
- "required": [
- "type",
- "rows"
- ],
- "properties": {
- "type": {
- "const": "table"
- },
- "header": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- },
- "rows": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- }
- },
- "text": {
- "type": "object",
- "description": "Raw text",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "text"
- },
- "value": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "url": {
- "type": "object",
- "description": "A single URL",
- "required": [
- "type",
- "href"
- ],
- "properties": {
- "type": {
- "const": "url"
- },
- "text": {
- "$ref": "#/definitions/text_value"
- },
- "href": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "http://mysite.com"
- ]
- }
- }
- },
- "code": {
- "type": "object",
- "description": "A codeblock",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "code"
- },
- "value": {
- "type": "string"
- },
- "lang": {
- "type": "string",
- "description": "A programming language"
- }
- }
- },
- "value": {
- "type": "object",
- "description": "A field that can store a range of types of value",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "value"
- },
- "value": {
- "type": [
- "number",
- "string",
- "boolean"
- ]
- }
- }
- },
- "diff": {
- "type": "object",
- "description": "A diff",
- "required": [
- "type",
- "before",
- "after"
- ],
- "properties": {
- "type": {
- "const": "diff"
- },
- "before": {
- "type": "string"
- },
- "after": {
- "type": "string"
- }
- }
- },
- "markdown": {
- "type": "object",
- "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "markdown"
- },
- "value": {
- "$ref": "#/definitions/text_value",
- "examples": [
- "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
- ]
- }
- }
- },
- "commit": {
- "type": "object",
- "description": "A commit/tag/branch within the GitLab project",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "commit"
- },
- "value": {
- "type": "string",
- "description": "The commit SHA",
- "minLength": 1
- }
- }
- },
- "file_location": {
- "type": "object",
- "description": "A location within a file in the project",
- "required": [
- "type",
- "file_name",
- "line_start"
- ],
- "properties": {
- "type": {
- "const": "file-location"
- },
- "file_name": {
- "type": "string",
- "minLength": 1
- },
- "line_start": {
- "type": "integer"
- },
- "line_end": {
- "type": "integer"
- }
- }
- },
- "module_location": {
- "type": "object",
- "description": "A location within a binary module of the form module+relative_offset",
- "required": [
- "type",
- "module_name",
- "offset"
- ],
- "properties": {
- "type": {
- "const": "module-location"
- },
- "module_name": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "compiled_binary"
- ]
- },
- "offset": {
- "type": "integer",
- "examples": [
- 100
- ]
- }
- }
- }
- },
- "self": {
- "version": "14.0.2"
- },
- "required": [
- "version",
- "vulnerabilities"
- ],
- "additionalProperties": true,
- "properties": {
- "scan": {
- "type": "object",
- "required": [
- "end_time",
- "scanned_resources",
- "scanner",
- "start_time",
- "status",
- "type"
- ],
- "properties": {
- "end_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-01-28T03:26:02"
- ]
- },
- "messages": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Communication intended for the initiator of a scan.",
- "required": [
- "level",
- "value"
- ],
- "properties": {
- "level": {
- "type": "string",
- "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
- "enum": [
- "info",
- "warn",
- "fatal"
- ],
- "examples": [
- "info"
- ]
- },
- "value": {
- "type": "string",
- "description": "The message to communicate.",
- "minLength": 1,
- "examples": [
- "Permission denied, scanning aborted"
- ]
- }
- }
- }
- },
- "analyzer": {
- "type": "object",
- "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the analyzer.",
- "minLength": 1,
- "examples": [
- "gitlab-dast"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the analyzer, not required to be unique.",
- "minLength": 1,
- "examples": [
- "GitLab DAST"
- ]
- },
- "url": {
- "type": "string",
- "format": "uri",
- "pattern": "^https?://.+",
- "description": "A link to more information about the analyzer.",
- "examples": [
- "https://docs.gitlab.com/ee/user/application_security/dast"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the analyzer.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- },
- "version": {
- "type": "string",
- "description": "The version of the analyzer.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- }
- }
- },
- "scanner": {
- "type": "object",
- "description": "Object defining the scanner used to perform the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the scanner.",
- "minLength": 1,
- "examples": [
- "my-sast-scanner"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the scanner, not required to be unique.",
- "minLength": 1,
- "examples": [
- "My SAST Scanner"
- ]
- },
- "url": {
- "type": "string",
- "description": "A link to more information about the scanner.",
- "examples": [
- "https://scanner.url"
- ]
- },
- "version": {
- "type": "string",
- "description": "The version of the scanner.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the scanner.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- }
- }
- },
- "start_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-02-14T16:01:59"
- ]
- },
- "status": {
- "type": "string",
- "description": "Result of the scan.",
- "enum": [
- "success",
- "failure"
- ]
- },
- "type": {
- "type": "string",
- "description": "Type of the scan.",
- "enum": [
- "dast",
- "api_fuzzing"
- ]
- },
- "scanned_resources": {
- "type": "array",
- "description": "The attack surface scanned by DAST.",
- "items": {
- "type": "object",
- "required": [
- "method",
- "url",
- "type"
- ],
- "properties": {
- "method": {
- "type": "string",
- "minLength": 1,
- "description": "HTTP method of the scanned resource.",
- "examples": [
- "GET",
- "POST",
- "HEAD"
- ]
- },
- "url": {
- "type": "string",
- "minLength": 1,
- "description": "URL of the scanned resource.",
- "examples": [
- "http://my.site.com/a-page"
- ]
- },
- "type": {
- "type": "string",
- "minLength": 1,
- "description": "Type of the scanned resource, for DAST, this must be 'url'.",
- "examples": [
- "url"
- ]
- }
- }
- }
- }
- }
- },
- "schema": {
- "type": "string",
- "description": "URI pointing to the validating security report schema.",
- "format": "uri"
- },
- "version": {
- "type": "string",
- "description": "The version of the schema to which the JSON report conforms.",
- "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
- },
- "vulnerabilities": {
- "type": "array",
- "description": "Array of vulnerability objects.",
- "items": {
- "type": "object",
- "description": "Describes the vulnerability.",
- "required": [
- "category",
- "cve",
- "identifiers",
- "location",
- "scanner"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
- "examples": [
- "642735a5-1425-428d-8d4e-3c854885a3c9"
- ]
- },
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
- "name": {
- "type": "string",
- "description": "The name of the vulnerability. This must not include the finding's specific information."
- },
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
- "description": {
- "type": "string",
- "description": "A long text section describing the vulnerability more fully."
- },
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
- "severity": {
- "type": "string",
- "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Info",
- "Unknown",
- "Low",
- "Medium",
- "High",
- "Critical"
- ]
- },
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
- "solution": {
- "type": "string",
- "description": "Explanation of how to fix the vulnerability."
- },
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
- "identifiers": {
- "type": "array",
- "minItems": 1,
- "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
- "items": {
- "type": "object",
- "required": [
- "type",
- "name",
- "value"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
- "minLength": 1
- },
- "name": {
- "type": "string",
- "description": "Human-readable name of the identifier.",
- "minLength": 1
- },
- "url": {
- "type": "string",
- "description": "URL of the identifier's documentation.",
- "format": "uri"
- },
- "value": {
- "type": "string",
- "description": "Value of the identifier, for matching purpose.",
- "minLength": 1
- }
- }
- }
- },
- "links": {
- "type": "array",
- "description": "An array of references to external documentation or articles that describe the vulnerability.",
- "items": {
- "type": "object",
- "required": [
- "url"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the vulnerability details link."
- },
- "url": {
- "type": "string",
- "description": "URL of the vulnerability details document.",
- "format": "uri"
- }
- }
- }
- },
- "details": {
- "$ref": "#/definitions/named_list/properties/items"
- },
- "tracking": {
- "description": "Describes how this vulnerability should be tracked as the project changes.",
- "oneOf": [
- {
- "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
- "required": [
- "items"
- ],
- "properties": {
- "type": {
- "const": "source"
- },
- "items": {
- "type": "array",
- "items": {
- "description": "An item that should be tracked using source-specific tracking methods.",
- "type": "object",
- "required": [
- "signatures"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the file that includes the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the file that includes the vulnerability."
- },
- "signatures": {
- "type": "array",
- "description": "An array of calculated tracking signatures for this tracking item.",
- "minItems": 1,
- "items": {
- "description": "A calculated tracking signature value and metadata.",
- "required": [
- "algorithm",
- "value"
- ],
- "properties": {
- "algorithm": {
- "type": "string",
- "description": "The algorithm used to generate the signature."
- },
- "value": {
- "type": "string",
- "description": "The result of this signature algorithm."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "Each tracking type must declare its own type."
- }
- }
- },
- "evidence": {
- "type": "object",
- "properties": {
- "source": {
- "type": "object",
- "description": "Source of evidence",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "Unique source identifier",
- "examples": [
- "assert:LogAnalysis",
- "assert:StatusCode"
- ]
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Source display name",
- "examples": [
- "Log Analysis",
- "Status Code"
- ]
- },
- "url": {
- "type": "string",
- "description": "Link to additional information",
- "examples": [
- "https://docs.gitlab.com/ee/development/integrations/secure.html"
- ]
- }
- }
- },
- "summary": {
- "type": "string",
- "description": "Human readable string containing evidence of the vulnerability.",
- "examples": [
- "Credit card 4111111111111111 found",
- "Server leaked information nginx/1.17.6"
- ]
- },
- "request": {
- "type": "object",
- "description": "An HTTP request.",
- "required": [
- "headers",
- "method",
- "url"
- ],
- "properties": {
- "headers": {
- "type": "array",
- "description": "HTTP headers present on the request.",
- "items": {
- "type": "object",
- "required": [
- "name",
- "value"
- ],
- "properties": {
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Name of the HTTP header.",
- "examples": [
- "Accept",
- "Content-Length",
- "Content-Type"
- ]
- },
- "value": {
- "type": "string",
- "minLength": 1,
- "description": "Value of the HTTP header.",
- "examples": [
- "*/*",
- "560",
- "application/json; charset=utf-8"
- ]
- }
- }
- }
- },
- "method": {
- "type": "string",
- "minLength": 1,
- "description": "HTTP method used in the request.",
- "examples": [
- "GET",
- "POST"
- ]
- },
- "url": {
- "type": "string",
- "minLength": 1,
- "description": "URL of the request.",
- "examples": [
- "http://my.site.com/vulnerable-endpoint?show-credit-card"
- ]
- },
- "body": {
- "type": "string",
- "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.",
- "examples": [
- "user=jsmith&first=%27&last=smith"
- ]
- }
- }
- },
- "response": {
- "type": "object",
- "description": "An HTTP response.",
- "required": [
- "headers",
- "reason_phrase",
- "status_code"
- ],
- "properties": {
- "headers": {
- "type": "array",
- "description": "HTTP headers present on the request.",
- "items": {
- "type": "object",
- "required": [
- "name",
- "value"
- ],
- "properties": {
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Name of the HTTP header.",
- "examples": [
- "Accept",
- "Content-Length",
- "Content-Type"
- ]
- },
- "value": {
- "type": "string",
- "minLength": 1,
- "description": "Value of the HTTP header.",
- "examples": [
- "*/*",
- "560",
- "application/json; charset=utf-8"
- ]
- }
- }
- }
- },
- "reason_phrase": {
- "type": "string",
- "description": "HTTP reason phrase of the response.",
- "examples": [
- "OK",
- "Internal Server Error"
- ]
- },
- "status_code": {
- "type": "integer",
- "description": "HTTP status code of the response.",
- "examples": [
- 200,
- 500
- ]
- },
- "body": {
- "type": "string",
- "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.",
- "examples": [
- "{\"user_id\": 2}"
- ]
- }
- }
- },
- "supporting_messages": {
- "type": "array",
- "description": "Array of supporting http messages.",
- "items": {
- "type": "object",
- "description": "A supporting http message.",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Message display name.",
- "examples": [
- "Unmodified",
- "Recorded"
- ]
- },
- "request": {
- "type": "object",
- "description": "An HTTP request.",
- "required": [
- "headers",
- "method",
- "url"
- ],
- "properties": {
- "headers": {
- "type": "array",
- "description": "HTTP headers present on the request.",
- "items": {
- "type": "object",
- "required": [
- "name",
- "value"
- ],
- "properties": {
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Name of the HTTP header.",
- "examples": [
- "Accept",
- "Content-Length",
- "Content-Type"
- ]
- },
- "value": {
- "type": "string",
- "minLength": 1,
- "description": "Value of the HTTP header.",
- "examples": [
- "*/*",
- "560",
- "application/json; charset=utf-8"
- ]
- }
- }
- }
- },
- "method": {
- "type": "string",
- "minLength": 1,
- "description": "HTTP method used in the request.",
- "examples": [
- "GET",
- "POST"
- ]
- },
- "url": {
- "type": "string",
- "minLength": 1,
- "description": "URL of the request.",
- "examples": [
- "http://my.site.com/vulnerable-endpoint?show-credit-card"
- ]
- },
- "body": {
- "type": "string",
- "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.",
- "examples": [
- "user=jsmith&first=%27&last=smith"
- ]
- }
- }
- },
- "response": {
- "type": "object",
- "description": "An HTTP response.",
- "required": [
- "headers",
- "reason_phrase",
- "status_code"
- ],
- "properties": {
- "headers": {
- "type": "array",
- "description": "HTTP headers present on the request.",
- "items": {
- "type": "object",
- "required": [
- "name",
- "value"
- ],
- "properties": {
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Name of the HTTP header.",
- "examples": [
- "Accept",
- "Content-Length",
- "Content-Type"
- ]
- },
- "value": {
- "type": "string",
- "minLength": 1,
- "description": "Value of the HTTP header.",
- "examples": [
- "*/*",
- "560",
- "application/json; charset=utf-8"
- ]
- }
- }
- }
- },
- "reason_phrase": {
- "type": "string",
- "description": "HTTP reason phrase of the response.",
- "examples": [
- "OK",
- "Internal Server Error"
- ]
- },
- "status_code": {
- "type": "integer",
- "description": "HTTP status code of the response.",
- "examples": [
- 200,
- 500
- ]
- },
- "body": {
- "type": "string",
- "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.",
- "examples": [
- "{\"user_id\": 2}"
- ]
- }
- }
- }
- }
- }
- }
- }
- },
- "location": {
- "type": "object",
- "description": "Identifies the vulnerability's location.",
- "properties": {
- "hostname": {
- "type": "string",
- "description": "The protocol, domain, and port of the application where the vulnerability was found."
- },
- "method": {
- "type": "string",
- "description": "The HTTP method that was used to request the URL where the vulnerability was found."
- },
- "param": {
- "type": "string",
- "description": "A value provided by a vulnerability rule related to the found vulnerability. Examples include a header value, or a parameter used in a HTTP POST."
- },
- "path": {
- "type": "string",
- "description": "The path of the URL where the vulnerability was found. Typically, this would start with a forward slash."
- }
- }
- },
- "assets": {
- "type": "array",
- "description": "Array of build assets associated with vulnerability.",
- "items": {
- "type": "object",
- "description": "Describes an asset associated with vulnerability.",
- "required": [
- "type",
- "name",
- "url"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "The type of asset",
- "enum": [
- "http_session",
- "postman"
- ]
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Display name for asset",
- "examples": [
- "HTTP Messages",
- "Postman Collection"
- ]
- },
- "url": {
- "type": "string",
- "minLength": 1,
- "description": "Link to asset in build artifacts",
- "examples": [
- "https://gitlab.com/gitlab-org/security-products/dast/-/jobs/626397001/artifacts/file//output/zap_session.data"
- ]
- }
- }
- }
- },
- "discovered_at": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss.sss, representing when the vulnerability was discovered",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}\\.\\d{3}$",
- "examples": [
- "2020-01-28T03:26:02.956"
- ]
- }
- }
- }
- },
- "remediations": {
- "type": "array",
- "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
- "items": {
- "type": "object",
- "required": [
- "fixes",
- "summary",
- "diff"
- ],
- "properties": {
- "fixes": {
- "type": "array",
- "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
- "items": {
- "type": "object",
- "required": [
- "cve"
- ],
- "properties": {
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- }
- }
- }
- },
- "summary": {
- "type": "string",
- "minLength": 1,
- "description": "An overview of how the vulnerabilities were fixed."
- },
- "diff": {
- "type": "string",
- "minLength": 1,
- "description": "A base64-encoded remediation code diff, compatible with git apply."
- }
- }
- }
- }
- }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.2/dependency-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.2/dependency-scanning-report-format.json
deleted file mode 100644
index d6ff5248358..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.2/dependency-scanning-report-format.json
+++ /dev/null
@@ -1,935 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Report format for GitLab Dependency Scanning",
- "description": "This schema provides the the report format for Dependency Scanning analyzers (https://docs.gitlab.com/ee/user/application_security/dependency_scanning).",
- "definitions": {
- "detail_type": {
- "oneOf": [
- {
- "$ref": "#/definitions/named_list"
- },
- {
- "$ref": "#/definitions/list"
- },
- {
- "$ref": "#/definitions/table"
- },
- {
- "$ref": "#/definitions/text"
- },
- {
- "$ref": "#/definitions/url"
- },
- {
- "$ref": "#/definitions/code"
- },
- {
- "$ref": "#/definitions/value"
- },
- {
- "$ref": "#/definitions/diff"
- },
- {
- "$ref": "#/definitions/markdown"
- },
- {
- "$ref": "#/definitions/commit"
- },
- {
- "$ref": "#/definitions/file_location"
- },
- {
- "$ref": "#/definitions/module_location"
- }
- ]
- },
- "text_value": {
- "type": "string"
- },
- "named_field": {
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "$ref": "#/definitions/text_value",
- "minLength": 1
- },
- "description": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "named_list": {
- "type": "object",
- "description": "An object with named and typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "named-list"
- },
- "items": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- {
- "$ref": "#/definitions/named_field"
- },
- {
- "$ref": "#/definitions/detail_type"
- }
- ]
- }
- }
- }
- }
- },
- "list": {
- "type": "object",
- "description": "A list of typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- },
- "table": {
- "type": "object",
- "description": "A table of typed fields",
- "required": [
- "type",
- "rows"
- ],
- "properties": {
- "type": {
- "const": "table"
- },
- "header": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- },
- "rows": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- }
- },
- "text": {
- "type": "object",
- "description": "Raw text",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "text"
- },
- "value": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "url": {
- "type": "object",
- "description": "A single URL",
- "required": [
- "type",
- "href"
- ],
- "properties": {
- "type": {
- "const": "url"
- },
- "text": {
- "$ref": "#/definitions/text_value"
- },
- "href": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "http://mysite.com"
- ]
- }
- }
- },
- "code": {
- "type": "object",
- "description": "A codeblock",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "code"
- },
- "value": {
- "type": "string"
- },
- "lang": {
- "type": "string",
- "description": "A programming language"
- }
- }
- },
- "value": {
- "type": "object",
- "description": "A field that can store a range of types of value",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "value"
- },
- "value": {
- "type": [
- "number",
- "string",
- "boolean"
- ]
- }
- }
- },
- "diff": {
- "type": "object",
- "description": "A diff",
- "required": [
- "type",
- "before",
- "after"
- ],
- "properties": {
- "type": {
- "const": "diff"
- },
- "before": {
- "type": "string"
- },
- "after": {
- "type": "string"
- }
- }
- },
- "markdown": {
- "type": "object",
- "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "markdown"
- },
- "value": {
- "$ref": "#/definitions/text_value",
- "examples": [
- "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
- ]
- }
- }
- },
- "commit": {
- "type": "object",
- "description": "A commit/tag/branch within the GitLab project",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "commit"
- },
- "value": {
- "type": "string",
- "description": "The commit SHA",
- "minLength": 1
- }
- }
- },
- "file_location": {
- "type": "object",
- "description": "A location within a file in the project",
- "required": [
- "type",
- "file_name",
- "line_start"
- ],
- "properties": {
- "type": {
- "const": "file-location"
- },
- "file_name": {
- "type": "string",
- "minLength": 1
- },
- "line_start": {
- "type": "integer"
- },
- "line_end": {
- "type": "integer"
- }
- }
- },
- "module_location": {
- "type": "object",
- "description": "A location within a binary module of the form module+relative_offset",
- "required": [
- "type",
- "module_name",
- "offset"
- ],
- "properties": {
- "type": {
- "const": "module-location"
- },
- "module_name": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "compiled_binary"
- ]
- },
- "offset": {
- "type": "integer",
- "examples": [
- 100
- ]
- }
- }
- }
- },
- "self": {
- "version": "14.0.2"
- },
- "required": [
- "dependency_files",
- "version",
- "vulnerabilities"
- ],
- "additionalProperties": true,
- "properties": {
- "scan": {
- "type": "object",
- "required": [
- "end_time",
- "scanner",
- "start_time",
- "status",
- "type"
- ],
- "properties": {
- "end_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-01-28T03:26:02"
- ]
- },
- "messages": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Communication intended for the initiator of a scan.",
- "required": [
- "level",
- "value"
- ],
- "properties": {
- "level": {
- "type": "string",
- "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
- "enum": [
- "info",
- "warn",
- "fatal"
- ],
- "examples": [
- "info"
- ]
- },
- "value": {
- "type": "string",
- "description": "The message to communicate.",
- "minLength": 1,
- "examples": [
- "Permission denied, scanning aborted"
- ]
- }
- }
- }
- },
- "analyzer": {
- "type": "object",
- "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the analyzer.",
- "minLength": 1,
- "examples": [
- "gitlab-dast"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the analyzer, not required to be unique.",
- "minLength": 1,
- "examples": [
- "GitLab DAST"
- ]
- },
- "url": {
- "type": "string",
- "format": "uri",
- "pattern": "^https?://.+",
- "description": "A link to more information about the analyzer.",
- "examples": [
- "https://docs.gitlab.com/ee/user/application_security/dast"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the analyzer.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- },
- "version": {
- "type": "string",
- "description": "The version of the analyzer.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- }
- }
- },
- "scanner": {
- "type": "object",
- "description": "Object defining the scanner used to perform the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the scanner.",
- "minLength": 1,
- "examples": [
- "my-sast-scanner"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the scanner, not required to be unique.",
- "minLength": 1,
- "examples": [
- "My SAST Scanner"
- ]
- },
- "url": {
- "type": "string",
- "description": "A link to more information about the scanner.",
- "examples": [
- "https://scanner.url"
- ]
- },
- "version": {
- "type": "string",
- "description": "The version of the scanner.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the scanner.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- }
- }
- },
- "start_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-02-14T16:01:59"
- ]
- },
- "status": {
- "type": "string",
- "description": "Result of the scan.",
- "enum": [
- "success",
- "failure"
- ]
- },
- "type": {
- "type": "string",
- "description": "Type of the scan.",
- "enum": [
- "dependency_scanning"
- ]
- }
- }
- },
- "schema": {
- "type": "string",
- "description": "URI pointing to the validating security report schema.",
- "format": "uri"
- },
- "version": {
- "type": "string",
- "description": "The version of the schema to which the JSON report conforms.",
- "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
- },
- "vulnerabilities": {
- "type": "array",
- "description": "Array of vulnerability objects.",
- "items": {
- "type": "object",
- "description": "Describes the vulnerability.",
- "required": [
- "category",
- "cve",
- "identifiers",
- "location",
- "scanner"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
- "examples": [
- "642735a5-1425-428d-8d4e-3c854885a3c9"
- ]
- },
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
- "name": {
- "type": "string",
- "description": "The name of the vulnerability. This must not include the finding's specific information."
- },
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
- "description": {
- "type": "string",
- "description": "A long text section describing the vulnerability more fully."
- },
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
- "severity": {
- "type": "string",
- "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Info",
- "Unknown",
- "Low",
- "Medium",
- "High",
- "Critical"
- ]
- },
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
- "solution": {
- "type": "string",
- "description": "Explanation of how to fix the vulnerability."
- },
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
- "identifiers": {
- "type": "array",
- "minItems": 1,
- "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
- "items": {
- "type": "object",
- "required": [
- "type",
- "name",
- "value"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
- "minLength": 1
- },
- "name": {
- "type": "string",
- "description": "Human-readable name of the identifier.",
- "minLength": 1
- },
- "url": {
- "type": "string",
- "description": "URL of the identifier's documentation.",
- "format": "uri"
- },
- "value": {
- "type": "string",
- "description": "Value of the identifier, for matching purpose.",
- "minLength": 1
- }
- }
- }
- },
- "links": {
- "type": "array",
- "description": "An array of references to external documentation or articles that describe the vulnerability.",
- "items": {
- "type": "object",
- "required": [
- "url"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the vulnerability details link."
- },
- "url": {
- "type": "string",
- "description": "URL of the vulnerability details document.",
- "format": "uri"
- }
- }
- }
- },
- "details": {
- "$ref": "#/definitions/named_list/properties/items"
- },
- "tracking": {
- "description": "Describes how this vulnerability should be tracked as the project changes.",
- "oneOf": [
- {
- "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
- "required": [
- "items"
- ],
- "properties": {
- "type": {
- "const": "source"
- },
- "items": {
- "type": "array",
- "items": {
- "description": "An item that should be tracked using source-specific tracking methods.",
- "type": "object",
- "required": [
- "signatures"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the file that includes the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the file that includes the vulnerability."
- },
- "signatures": {
- "type": "array",
- "description": "An array of calculated tracking signatures for this tracking item.",
- "minItems": 1,
- "items": {
- "description": "A calculated tracking signature value and metadata.",
- "required": [
- "algorithm",
- "value"
- ],
- "properties": {
- "algorithm": {
- "type": "string",
- "description": "The algorithm used to generate the signature."
- },
- "value": {
- "type": "string",
- "description": "The result of this signature algorithm."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "Each tracking type must declare its own type."
- }
- }
- },
- "location": {
- "type": "object",
- "description": "Identifies the vulnerability's location.",
- "required": [
- "file",
- "dependency"
- ],
- "properties": {
- "file": {
- "type": "string",
- "minLength": 1,
- "description": "Path to the manifest or lock file where the dependency is declared (such as yarn.lock)."
- },
- "dependency": {
- "type": "object",
- "description": "Describes the dependency of a project where the vulnerability is located.",
- "properties": {
- "package": {
- "type": "object",
- "description": "Provides information on the package where the vulnerability is located.",
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the package where the vulnerability is located."
- }
- }
- },
- "version": {
- "type": "string",
- "description": "Version of the vulnerable package."
- },
- "iid": {
- "description": "ID that identifies the dependency in the scope of a dependency file.",
- "type": "number"
- },
- "direct": {
- "type": "boolean",
- "description": "Tells whether this is a direct, top-level dependency of the scanned project."
- },
- "dependency_path": {
- "type": "array",
- "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.",
- "items": {
- "type": "object",
- "required": [
- "iid"
- ],
- "properties": {
- "iid": {
- "type": "number",
- "description": "ID that is unique in the scope of a parent object, and specific to the resource type."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- }
- },
- "remediations": {
- "type": "array",
- "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
- "items": {
- "type": "object",
- "required": [
- "fixes",
- "summary",
- "diff"
- ],
- "properties": {
- "fixes": {
- "type": "array",
- "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
- "items": {
- "type": "object",
- "required": [
- "cve"
- ],
- "properties": {
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- }
- }
- }
- },
- "summary": {
- "type": "string",
- "minLength": 1,
- "description": "An overview of how the vulnerabilities were fixed."
- },
- "diff": {
- "type": "string",
- "minLength": 1,
- "description": "A base64-encoded remediation code diff, compatible with git apply."
- }
- }
- }
- },
- "dependency_files": {
- "type": "array",
- "description": "List of dependency files identified in the project.",
- "items": {
- "type": "object",
- "required": [
- "path",
- "package_manager",
- "dependencies"
- ],
- "properties": {
- "path": {
- "type": "string",
- "minLength": 1
- },
- "package_manager": {
- "type": "string",
- "minLength": 1
- },
- "dependencies": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Describes the dependency of a project where the vulnerability is located.",
- "properties": {
- "package": {
- "type": "object",
- "description": "Provides information on the package where the vulnerability is located.",
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the package where the vulnerability is located."
- }
- }
- },
- "version": {
- "type": "string",
- "description": "Version of the vulnerable package."
- },
- "iid": {
- "description": "ID that identifies the dependency in the scope of a dependency file.",
- "type": "number"
- },
- "direct": {
- "type": "boolean",
- "description": "Tells whether this is a direct, top-level dependency of the scanned project."
- },
- "dependency_path": {
- "type": "array",
- "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.",
- "items": {
- "type": "object",
- "required": [
- "iid"
- ],
- "properties": {
- "iid": {
- "type": "number",
- "description": "ID that is unique in the scope of a parent object, and specific to the resource type."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- }
- }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.2/sast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.2/sast-report-format.json
deleted file mode 100644
index 2be6801d2f6..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.2/sast-report-format.json
+++ /dev/null
@@ -1,836 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Report format for GitLab SAST",
- "description": "This schema provides the report format for Static Application Security Testing analyzers (https://docs.gitlab.com/ee/user/application_security/sast).",
- "definitions": {
- "detail_type": {
- "oneOf": [
- {
- "$ref": "#/definitions/named_list"
- },
- {
- "$ref": "#/definitions/list"
- },
- {
- "$ref": "#/definitions/table"
- },
- {
- "$ref": "#/definitions/text"
- },
- {
- "$ref": "#/definitions/url"
- },
- {
- "$ref": "#/definitions/code"
- },
- {
- "$ref": "#/definitions/value"
- },
- {
- "$ref": "#/definitions/diff"
- },
- {
- "$ref": "#/definitions/markdown"
- },
- {
- "$ref": "#/definitions/commit"
- },
- {
- "$ref": "#/definitions/file_location"
- },
- {
- "$ref": "#/definitions/module_location"
- }
- ]
- },
- "text_value": {
- "type": "string"
- },
- "named_field": {
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "$ref": "#/definitions/text_value",
- "minLength": 1
- },
- "description": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "named_list": {
- "type": "object",
- "description": "An object with named and typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "named-list"
- },
- "items": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- {
- "$ref": "#/definitions/named_field"
- },
- {
- "$ref": "#/definitions/detail_type"
- }
- ]
- }
- }
- }
- }
- },
- "list": {
- "type": "object",
- "description": "A list of typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- },
- "table": {
- "type": "object",
- "description": "A table of typed fields",
- "required": [
- "type",
- "rows"
- ],
- "properties": {
- "type": {
- "const": "table"
- },
- "header": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- },
- "rows": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- }
- },
- "text": {
- "type": "object",
- "description": "Raw text",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "text"
- },
- "value": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "url": {
- "type": "object",
- "description": "A single URL",
- "required": [
- "type",
- "href"
- ],
- "properties": {
- "type": {
- "const": "url"
- },
- "text": {
- "$ref": "#/definitions/text_value"
- },
- "href": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "http://mysite.com"
- ]
- }
- }
- },
- "code": {
- "type": "object",
- "description": "A codeblock",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "code"
- },
- "value": {
- "type": "string"
- },
- "lang": {
- "type": "string",
- "description": "A programming language"
- }
- }
- },
- "value": {
- "type": "object",
- "description": "A field that can store a range of types of value",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "value"
- },
- "value": {
- "type": [
- "number",
- "string",
- "boolean"
- ]
- }
- }
- },
- "diff": {
- "type": "object",
- "description": "A diff",
- "required": [
- "type",
- "before",
- "after"
- ],
- "properties": {
- "type": {
- "const": "diff"
- },
- "before": {
- "type": "string"
- },
- "after": {
- "type": "string"
- }
- }
- },
- "markdown": {
- "type": "object",
- "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "markdown"
- },
- "value": {
- "$ref": "#/definitions/text_value",
- "examples": [
- "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
- ]
- }
- }
- },
- "commit": {
- "type": "object",
- "description": "A commit/tag/branch within the GitLab project",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "commit"
- },
- "value": {
- "type": "string",
- "description": "The commit SHA",
- "minLength": 1
- }
- }
- },
- "file_location": {
- "type": "object",
- "description": "A location within a file in the project",
- "required": [
- "type",
- "file_name",
- "line_start"
- ],
- "properties": {
- "type": {
- "const": "file-location"
- },
- "file_name": {
- "type": "string",
- "minLength": 1
- },
- "line_start": {
- "type": "integer"
- },
- "line_end": {
- "type": "integer"
- }
- }
- },
- "module_location": {
- "type": "object",
- "description": "A location within a binary module of the form module+relative_offset",
- "required": [
- "type",
- "module_name",
- "offset"
- ],
- "properties": {
- "type": {
- "const": "module-location"
- },
- "module_name": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "compiled_binary"
- ]
- },
- "offset": {
- "type": "integer",
- "examples": [
- 100
- ]
- }
- }
- }
- },
- "self": {
- "version": "14.0.2"
- },
- "required": [
- "version",
- "vulnerabilities"
- ],
- "additionalProperties": true,
- "properties": {
- "scan": {
- "type": "object",
- "required": [
- "end_time",
- "scanner",
- "start_time",
- "status",
- "type"
- ],
- "properties": {
- "end_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-01-28T03:26:02"
- ]
- },
- "messages": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Communication intended for the initiator of a scan.",
- "required": [
- "level",
- "value"
- ],
- "properties": {
- "level": {
- "type": "string",
- "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
- "enum": [
- "info",
- "warn",
- "fatal"
- ],
- "examples": [
- "info"
- ]
- },
- "value": {
- "type": "string",
- "description": "The message to communicate.",
- "minLength": 1,
- "examples": [
- "Permission denied, scanning aborted"
- ]
- }
- }
- }
- },
- "analyzer": {
- "type": "object",
- "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the analyzer.",
- "minLength": 1,
- "examples": [
- "gitlab-dast"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the analyzer, not required to be unique.",
- "minLength": 1,
- "examples": [
- "GitLab DAST"
- ]
- },
- "url": {
- "type": "string",
- "format": "uri",
- "pattern": "^https?://.+",
- "description": "A link to more information about the analyzer.",
- "examples": [
- "https://docs.gitlab.com/ee/user/application_security/dast"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the analyzer.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- },
- "version": {
- "type": "string",
- "description": "The version of the analyzer.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- }
- }
- },
- "scanner": {
- "type": "object",
- "description": "Object defining the scanner used to perform the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the scanner.",
- "minLength": 1,
- "examples": [
- "my-sast-scanner"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the scanner, not required to be unique.",
- "minLength": 1,
- "examples": [
- "My SAST Scanner"
- ]
- },
- "url": {
- "type": "string",
- "description": "A link to more information about the scanner.",
- "examples": [
- "https://scanner.url"
- ]
- },
- "version": {
- "type": "string",
- "description": "The version of the scanner.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the scanner.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- }
- }
- },
- "start_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-02-14T16:01:59"
- ]
- },
- "status": {
- "type": "string",
- "description": "Result of the scan.",
- "enum": [
- "success",
- "failure"
- ]
- },
- "type": {
- "type": "string",
- "description": "Type of the scan.",
- "enum": [
- "sast"
- ]
- }
- }
- },
- "schema": {
- "type": "string",
- "description": "URI pointing to the validating security report schema.",
- "format": "uri"
- },
- "version": {
- "type": "string",
- "description": "The version of the schema to which the JSON report conforms.",
- "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
- },
- "vulnerabilities": {
- "type": "array",
- "description": "Array of vulnerability objects.",
- "items": {
- "type": "object",
- "description": "Describes the vulnerability.",
- "required": [
- "category",
- "cve",
- "identifiers",
- "location",
- "scanner"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
- "examples": [
- "642735a5-1425-428d-8d4e-3c854885a3c9"
- ]
- },
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
- "name": {
- "type": "string",
- "description": "The name of the vulnerability. This must not include the finding's specific information."
- },
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
- "description": {
- "type": "string",
- "description": "A long text section describing the vulnerability more fully."
- },
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
- "severity": {
- "type": "string",
- "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Info",
- "Unknown",
- "Low",
- "Medium",
- "High",
- "Critical"
- ]
- },
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
- "solution": {
- "type": "string",
- "description": "Explanation of how to fix the vulnerability."
- },
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
- "identifiers": {
- "type": "array",
- "minItems": 1,
- "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
- "items": {
- "type": "object",
- "required": [
- "type",
- "name",
- "value"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
- "minLength": 1
- },
- "name": {
- "type": "string",
- "description": "Human-readable name of the identifier.",
- "minLength": 1
- },
- "url": {
- "type": "string",
- "description": "URL of the identifier's documentation.",
- "format": "uri"
- },
- "value": {
- "type": "string",
- "description": "Value of the identifier, for matching purpose.",
- "minLength": 1
- }
- }
- }
- },
- "links": {
- "type": "array",
- "description": "An array of references to external documentation or articles that describe the vulnerability.",
- "items": {
- "type": "object",
- "required": [
- "url"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the vulnerability details link."
- },
- "url": {
- "type": "string",
- "description": "URL of the vulnerability details document.",
- "format": "uri"
- }
- }
- }
- },
- "details": {
- "$ref": "#/definitions/named_list/properties/items"
- },
- "tracking": {
- "description": "Describes how this vulnerability should be tracked as the project changes.",
- "oneOf": [
- {
- "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
- "required": [
- "items"
- ],
- "properties": {
- "type": {
- "const": "source"
- },
- "items": {
- "type": "array",
- "items": {
- "description": "An item that should be tracked using source-specific tracking methods.",
- "type": "object",
- "required": [
- "signatures"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the file that includes the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the file that includes the vulnerability."
- },
- "signatures": {
- "type": "array",
- "description": "An array of calculated tracking signatures for this tracking item.",
- "minItems": 1,
- "items": {
- "description": "A calculated tracking signature value and metadata.",
- "required": [
- "algorithm",
- "value"
- ],
- "properties": {
- "algorithm": {
- "type": "string",
- "description": "The algorithm used to generate the signature."
- },
- "value": {
- "type": "string",
- "description": "The result of this signature algorithm."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "Each tracking type must declare its own type."
- }
- }
- },
- "location": {
- "type": "object",
- "description": "Identifies the vulnerability's location.",
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the code affected by the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the code affected by the vulnerability."
- },
- "class": {
- "type": "string",
- "description": "Provides the name of the class where the vulnerability is located."
- },
- "method": {
- "type": "string",
- "description": "Provides the name of the method where the vulnerability is located."
- }
- }
- },
- "raw_source_code_extract": {
- "type": "string",
- "description": "Provides an unsanitized excerpt of the affected source code."
- }
- }
- }
- },
- "remediations": {
- "type": "array",
- "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
- "items": {
- "type": "object",
- "required": [
- "fixes",
- "summary",
- "diff"
- ],
- "properties": {
- "fixes": {
- "type": "array",
- "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
- "items": {
- "type": "object",
- "required": [
- "cve"
- ],
- "properties": {
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- }
- }
- }
- },
- "summary": {
- "type": "string",
- "minLength": 1,
- "description": "An overview of how the vulnerabilities were fixed."
- },
- "diff": {
- "type": "string",
- "minLength": 1,
- "description": "A base64-encoded remediation code diff, compatible with git apply."
- }
- }
- }
- }
- }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.2/secret-detection-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.2/secret-detection-report-format.json
deleted file mode 100644
index c44554489ce..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.2/secret-detection-report-format.json
+++ /dev/null
@@ -1,859 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Report format for GitLab Secret Detection",
- "description": "This schema provides the the report format for the Secret Detection analyzer (https://docs.gitlab.com/ee/user/application_security/secret_detection)",
- "definitions": {
- "detail_type": {
- "oneOf": [
- {
- "$ref": "#/definitions/named_list"
- },
- {
- "$ref": "#/definitions/list"
- },
- {
- "$ref": "#/definitions/table"
- },
- {
- "$ref": "#/definitions/text"
- },
- {
- "$ref": "#/definitions/url"
- },
- {
- "$ref": "#/definitions/code"
- },
- {
- "$ref": "#/definitions/value"
- },
- {
- "$ref": "#/definitions/diff"
- },
- {
- "$ref": "#/definitions/markdown"
- },
- {
- "$ref": "#/definitions/commit"
- },
- {
- "$ref": "#/definitions/file_location"
- },
- {
- "$ref": "#/definitions/module_location"
- }
- ]
- },
- "text_value": {
- "type": "string"
- },
- "named_field": {
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "$ref": "#/definitions/text_value",
- "minLength": 1
- },
- "description": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "named_list": {
- "type": "object",
- "description": "An object with named and typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "named-list"
- },
- "items": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- {
- "$ref": "#/definitions/named_field"
- },
- {
- "$ref": "#/definitions/detail_type"
- }
- ]
- }
- }
- }
- }
- },
- "list": {
- "type": "object",
- "description": "A list of typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- },
- "table": {
- "type": "object",
- "description": "A table of typed fields",
- "required": [
- "type",
- "rows"
- ],
- "properties": {
- "type": {
- "const": "table"
- },
- "header": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- },
- "rows": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- }
- },
- "text": {
- "type": "object",
- "description": "Raw text",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "text"
- },
- "value": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "url": {
- "type": "object",
- "description": "A single URL",
- "required": [
- "type",
- "href"
- ],
- "properties": {
- "type": {
- "const": "url"
- },
- "text": {
- "$ref": "#/definitions/text_value"
- },
- "href": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "http://mysite.com"
- ]
- }
- }
- },
- "code": {
- "type": "object",
- "description": "A codeblock",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "code"
- },
- "value": {
- "type": "string"
- },
- "lang": {
- "type": "string",
- "description": "A programming language"
- }
- }
- },
- "value": {
- "type": "object",
- "description": "A field that can store a range of types of value",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "value"
- },
- "value": {
- "type": [
- "number",
- "string",
- "boolean"
- ]
- }
- }
- },
- "diff": {
- "type": "object",
- "description": "A diff",
- "required": [
- "type",
- "before",
- "after"
- ],
- "properties": {
- "type": {
- "const": "diff"
- },
- "before": {
- "type": "string"
- },
- "after": {
- "type": "string"
- }
- }
- },
- "markdown": {
- "type": "object",
- "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "markdown"
- },
- "value": {
- "$ref": "#/definitions/text_value",
- "examples": [
- "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
- ]
- }
- }
- },
- "commit": {
- "type": "object",
- "description": "A commit/tag/branch within the GitLab project",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "commit"
- },
- "value": {
- "type": "string",
- "description": "The commit SHA",
- "minLength": 1
- }
- }
- },
- "file_location": {
- "type": "object",
- "description": "A location within a file in the project",
- "required": [
- "type",
- "file_name",
- "line_start"
- ],
- "properties": {
- "type": {
- "const": "file-location"
- },
- "file_name": {
- "type": "string",
- "minLength": 1
- },
- "line_start": {
- "type": "integer"
- },
- "line_end": {
- "type": "integer"
- }
- }
- },
- "module_location": {
- "type": "object",
- "description": "A location within a binary module of the form module+relative_offset",
- "required": [
- "type",
- "module_name",
- "offset"
- ],
- "properties": {
- "type": {
- "const": "module-location"
- },
- "module_name": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "compiled_binary"
- ]
- },
- "offset": {
- "type": "integer",
- "examples": [
- 100
- ]
- }
- }
- }
- },
- "self": {
- "version": "14.0.2"
- },
- "required": [
- "version",
- "vulnerabilities"
- ],
- "additionalProperties": true,
- "properties": {
- "scan": {
- "type": "object",
- "required": [
- "end_time",
- "scanner",
- "start_time",
- "status",
- "type"
- ],
- "properties": {
- "end_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-01-28T03:26:02"
- ]
- },
- "messages": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Communication intended for the initiator of a scan.",
- "required": [
- "level",
- "value"
- ],
- "properties": {
- "level": {
- "type": "string",
- "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
- "enum": [
- "info",
- "warn",
- "fatal"
- ],
- "examples": [
- "info"
- ]
- },
- "value": {
- "type": "string",
- "description": "The message to communicate.",
- "minLength": 1,
- "examples": [
- "Permission denied, scanning aborted"
- ]
- }
- }
- }
- },
- "analyzer": {
- "type": "object",
- "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the analyzer.",
- "minLength": 1,
- "examples": [
- "gitlab-dast"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the analyzer, not required to be unique.",
- "minLength": 1,
- "examples": [
- "GitLab DAST"
- ]
- },
- "url": {
- "type": "string",
- "format": "uri",
- "pattern": "^https?://.+",
- "description": "A link to more information about the analyzer.",
- "examples": [
- "https://docs.gitlab.com/ee/user/application_security/dast"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the analyzer.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- },
- "version": {
- "type": "string",
- "description": "The version of the analyzer.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- }
- }
- },
- "scanner": {
- "type": "object",
- "description": "Object defining the scanner used to perform the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the scanner.",
- "minLength": 1,
- "examples": [
- "my-sast-scanner"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the scanner, not required to be unique.",
- "minLength": 1,
- "examples": [
- "My SAST Scanner"
- ]
- },
- "url": {
- "type": "string",
- "description": "A link to more information about the scanner.",
- "examples": [
- "https://scanner.url"
- ]
- },
- "version": {
- "type": "string",
- "description": "The version of the scanner.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the scanner.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- }
- }
- },
- "start_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-02-14T16:01:59"
- ]
- },
- "status": {
- "type": "string",
- "description": "Result of the scan.",
- "enum": [
- "success",
- "failure"
- ]
- },
- "type": {
- "type": "string",
- "description": "Type of the scan.",
- "enum": [
- "secret_detection"
- ]
- }
- }
- },
- "schema": {
- "type": "string",
- "description": "URI pointing to the validating security report schema.",
- "format": "uri"
- },
- "version": {
- "type": "string",
- "description": "The version of the schema to which the JSON report conforms.",
- "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
- },
- "vulnerabilities": {
- "type": "array",
- "description": "Array of vulnerability objects.",
- "items": {
- "type": "object",
- "description": "Describes the vulnerability.",
- "required": [
- "category",
- "cve",
- "identifiers",
- "location",
- "scanner"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
- "examples": [
- "642735a5-1425-428d-8d4e-3c854885a3c9"
- ]
- },
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
- "name": {
- "type": "string",
- "description": "The name of the vulnerability. This must not include the finding's specific information."
- },
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
- "description": {
- "type": "string",
- "description": "A long text section describing the vulnerability more fully."
- },
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
- "severity": {
- "type": "string",
- "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Info",
- "Unknown",
- "Low",
- "Medium",
- "High",
- "Critical"
- ]
- },
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
- "solution": {
- "type": "string",
- "description": "Explanation of how to fix the vulnerability."
- },
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
- "identifiers": {
- "type": "array",
- "minItems": 1,
- "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
- "items": {
- "type": "object",
- "required": [
- "type",
- "name",
- "value"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
- "minLength": 1
- },
- "name": {
- "type": "string",
- "description": "Human-readable name of the identifier.",
- "minLength": 1
- },
- "url": {
- "type": "string",
- "description": "URL of the identifier's documentation.",
- "format": "uri"
- },
- "value": {
- "type": "string",
- "description": "Value of the identifier, for matching purpose.",
- "minLength": 1
- }
- }
- }
- },
- "links": {
- "type": "array",
- "description": "An array of references to external documentation or articles that describe the vulnerability.",
- "items": {
- "type": "object",
- "required": [
- "url"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the vulnerability details link."
- },
- "url": {
- "type": "string",
- "description": "URL of the vulnerability details document.",
- "format": "uri"
- }
- }
- }
- },
- "details": {
- "$ref": "#/definitions/named_list/properties/items"
- },
- "tracking": {
- "description": "Describes how this vulnerability should be tracked as the project changes.",
- "oneOf": [
- {
- "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
- "required": [
- "items"
- ],
- "properties": {
- "type": {
- "const": "source"
- },
- "items": {
- "type": "array",
- "items": {
- "description": "An item that should be tracked using source-specific tracking methods.",
- "type": "object",
- "required": [
- "signatures"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the file that includes the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the file that includes the vulnerability."
- },
- "signatures": {
- "type": "array",
- "description": "An array of calculated tracking signatures for this tracking item.",
- "minItems": 1,
- "items": {
- "description": "A calculated tracking signature value and metadata.",
- "required": [
- "algorithm",
- "value"
- ],
- "properties": {
- "algorithm": {
- "type": "string",
- "description": "The algorithm used to generate the signature."
- },
- "value": {
- "type": "string",
- "description": "The result of this signature algorithm."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "Each tracking type must declare its own type."
- }
- }
- },
- "location": {
- "required": [
- "commit"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located"
- },
- "commit": {
- "type": "object",
- "description": "Represents the commit in which the vulnerability was detected",
- "required": [
- "sha"
- ],
- "properties": {
- "author": {
- "type": "string"
- },
- "date": {
- "type": "string"
- },
- "message": {
- "type": "string"
- },
- "sha": {
- "type": "string",
- "minLength": 1
- }
- }
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the code affected by the vulnerability"
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the code affected by the vulnerability"
- },
- "class": {
- "type": "string",
- "description": "Provides the name of the class where the vulnerability is located"
- },
- "method": {
- "type": "string",
- "description": "Provides the name of the method where the vulnerability is located"
- }
- }
- },
- "raw_source_code_extract": {
- "type": "string",
- "description": "Provides an unsanitized excerpt of the affected source code."
- }
- }
- }
- },
- "remediations": {
- "type": "array",
- "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
- "items": {
- "type": "object",
- "required": [
- "fixes",
- "summary",
- "diff"
- ],
- "properties": {
- "fixes": {
- "type": "array",
- "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
- "items": {
- "type": "object",
- "required": [
- "cve"
- ],
- "properties": {
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- }
- }
- }
- },
- "summary": {
- "type": "string",
- "minLength": 1,
- "description": "An overview of how the vulnerabilities were fixed."
- },
- "diff": {
- "type": "string",
- "minLength": 1,
- "description": "A base64-encoded remediation code diff, compatible with git apply."
- }
- }
- }
- }
- }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.3/container-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.3/container-scanning-report-format.json
deleted file mode 100644
index 959b7b8f6f2..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.3/container-scanning-report-format.json
+++ /dev/null
@@ -1,904 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Report format for GitLab Container Scanning",
- "description": "This schema provides the the report format for Container Scanning (https://docs.gitlab.com/ee/user/application_security/container_scanning).",
- "definitions": {
- "detail_type": {
- "oneOf": [
- {
- "$ref": "#/definitions/named_list"
- },
- {
- "$ref": "#/definitions/list"
- },
- {
- "$ref": "#/definitions/table"
- },
- {
- "$ref": "#/definitions/text"
- },
- {
- "$ref": "#/definitions/url"
- },
- {
- "$ref": "#/definitions/code"
- },
- {
- "$ref": "#/definitions/value"
- },
- {
- "$ref": "#/definitions/diff"
- },
- {
- "$ref": "#/definitions/markdown"
- },
- {
- "$ref": "#/definitions/commit"
- },
- {
- "$ref": "#/definitions/file_location"
- },
- {
- "$ref": "#/definitions/module_location"
- }
- ]
- },
- "text_value": {
- "type": "string"
- },
- "named_field": {
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "$ref": "#/definitions/text_value",
- "minLength": 1
- },
- "description": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "named_list": {
- "type": "object",
- "description": "An object with named and typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "named-list"
- },
- "items": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- {
- "$ref": "#/definitions/named_field"
- },
- {
- "$ref": "#/definitions/detail_type"
- }
- ]
- }
- }
- }
- }
- },
- "list": {
- "type": "object",
- "description": "A list of typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- },
- "table": {
- "type": "object",
- "description": "A table of typed fields",
- "required": [
- "type",
- "rows"
- ],
- "properties": {
- "type": {
- "const": "table"
- },
- "header": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- },
- "rows": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- }
- },
- "text": {
- "type": "object",
- "description": "Raw text",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "text"
- },
- "value": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "url": {
- "type": "object",
- "description": "A single URL",
- "required": [
- "type",
- "href"
- ],
- "properties": {
- "type": {
- "const": "url"
- },
- "text": {
- "$ref": "#/definitions/text_value"
- },
- "href": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "http://mysite.com"
- ]
- }
- }
- },
- "code": {
- "type": "object",
- "description": "A codeblock",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "code"
- },
- "value": {
- "type": "string"
- },
- "lang": {
- "type": "string",
- "description": "A programming language"
- }
- }
- },
- "value": {
- "type": "object",
- "description": "A field that can store a range of types of value",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "value"
- },
- "value": {
- "type": [
- "number",
- "string",
- "boolean"
- ]
- }
- }
- },
- "diff": {
- "type": "object",
- "description": "A diff",
- "required": [
- "type",
- "before",
- "after"
- ],
- "properties": {
- "type": {
- "const": "diff"
- },
- "before": {
- "type": "string"
- },
- "after": {
- "type": "string"
- }
- }
- },
- "markdown": {
- "type": "object",
- "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "markdown"
- },
- "value": {
- "$ref": "#/definitions/text_value",
- "examples": [
- "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
- ]
- }
- }
- },
- "commit": {
- "type": "object",
- "description": "A commit/tag/branch within the GitLab project",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "commit"
- },
- "value": {
- "type": "string",
- "description": "The commit SHA",
- "minLength": 1
- }
- }
- },
- "file_location": {
- "type": "object",
- "description": "A location within a file in the project",
- "required": [
- "type",
- "file_name",
- "line_start"
- ],
- "properties": {
- "type": {
- "const": "file-location"
- },
- "file_name": {
- "type": "string",
- "minLength": 1
- },
- "line_start": {
- "type": "integer"
- },
- "line_end": {
- "type": "integer"
- }
- }
- },
- "module_location": {
- "type": "object",
- "description": "A location within a binary module of the form module+relative_offset",
- "required": [
- "type",
- "module_name",
- "offset"
- ],
- "properties": {
- "type": {
- "const": "module-location"
- },
- "module_name": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "compiled_binary"
- ]
- },
- "offset": {
- "type": "integer",
- "examples": [
- 100
- ]
- }
- }
- }
- },
- "self": {
- "version": "14.0.3"
- },
- "required": [
- "version",
- "vulnerabilities"
- ],
- "additionalProperties": true,
- "properties": {
- "scan": {
- "type": "object",
- "required": [
- "end_time",
- "scanner",
- "start_time",
- "status",
- "type"
- ],
- "properties": {
- "end_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-01-28T03:26:02"
- ]
- },
- "messages": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Communication intended for the initiator of a scan.",
- "required": [
- "level",
- "value"
- ],
- "properties": {
- "level": {
- "type": "string",
- "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
- "enum": [
- "info",
- "warn",
- "fatal"
- ],
- "examples": [
- "info"
- ]
- },
- "value": {
- "type": "string",
- "description": "The message to communicate.",
- "minLength": 1,
- "examples": [
- "Permission denied, scanning aborted"
- ]
- }
- }
- }
- },
- "analyzer": {
- "type": "object",
- "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the analyzer.",
- "minLength": 1,
- "examples": [
- "gitlab-dast"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the analyzer, not required to be unique.",
- "minLength": 1,
- "examples": [
- "GitLab DAST"
- ]
- },
- "url": {
- "type": "string",
- "format": "uri",
- "pattern": "^https?://.+",
- "description": "A link to more information about the analyzer.",
- "examples": [
- "https://docs.gitlab.com/ee/user/application_security/dast"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the analyzer.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- },
- "version": {
- "type": "string",
- "description": "The version of the analyzer.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- }
- }
- },
- "scanner": {
- "type": "object",
- "description": "Object defining the scanner used to perform the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the scanner.",
- "minLength": 1,
- "examples": [
- "my-sast-scanner"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the scanner, not required to be unique.",
- "minLength": 1,
- "examples": [
- "My SAST Scanner"
- ]
- },
- "url": {
- "type": "string",
- "description": "A link to more information about the scanner.",
- "examples": [
- "https://scanner.url"
- ]
- },
- "version": {
- "type": "string",
- "description": "The version of the scanner.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the scanner.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- }
- }
- },
- "start_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-02-14T16:01:59"
- ]
- },
- "status": {
- "type": "string",
- "description": "Result of the scan.",
- "enum": [
- "success",
- "failure"
- ]
- },
- "type": {
- "type": "string",
- "description": "Type of the scan.",
- "enum": [
- "container_scanning"
- ]
- }
- }
- },
- "schema": {
- "type": "string",
- "description": "URI pointing to the validating security report schema.",
- "format": "uri"
- },
- "version": {
- "type": "string",
- "description": "The version of the schema to which the JSON report conforms.",
- "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
- },
- "vulnerabilities": {
- "type": "array",
- "description": "Array of vulnerability objects.",
- "items": {
- "type": "object",
- "description": "Describes the vulnerability.",
- "required": [
- "category",
- "cve",
- "identifiers",
- "location",
- "scanner"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
- "examples": [
- "642735a5-1425-428d-8d4e-3c854885a3c9"
- ]
- },
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
- "name": {
- "type": "string",
- "description": "The name of the vulnerability. This must not include the finding's specific information."
- },
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
- "description": {
- "type": "string",
- "description": "A long text section describing the vulnerability more fully."
- },
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
- "severity": {
- "type": "string",
- "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Info",
- "Unknown",
- "Low",
- "Medium",
- "High",
- "Critical"
- ]
- },
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
- "solution": {
- "type": "string",
- "description": "Explanation of how to fix the vulnerability."
- },
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
- "identifiers": {
- "type": "array",
- "minItems": 1,
- "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
- "items": {
- "type": "object",
- "required": [
- "type",
- "name",
- "value"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
- "minLength": 1
- },
- "name": {
- "type": "string",
- "description": "Human-readable name of the identifier.",
- "minLength": 1
- },
- "url": {
- "type": "string",
- "description": "URL of the identifier's documentation.",
- "format": "uri"
- },
- "value": {
- "type": "string",
- "description": "Value of the identifier, for matching purpose.",
- "minLength": 1
- }
- }
- }
- },
- "links": {
- "type": "array",
- "description": "An array of references to external documentation or articles that describe the vulnerability.",
- "items": {
- "type": "object",
- "required": [
- "url"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the vulnerability details link."
- },
- "url": {
- "type": "string",
- "description": "URL of the vulnerability details document.",
- "format": "uri"
- }
- }
- }
- },
- "details": {
- "$ref": "#/definitions/named_list/properties/items"
- },
- "tracking": {
- "description": "Describes how this vulnerability should be tracked as the project changes.",
- "oneOf": [
- {
- "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
- "required": [
- "items"
- ],
- "properties": {
- "type": {
- "const": "source"
- },
- "items": {
- "type": "array",
- "items": {
- "description": "An item that should be tracked using source-specific tracking methods.",
- "type": "object",
- "required": [
- "signatures"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the file that includes the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the file that includes the vulnerability."
- },
- "signatures": {
- "type": "array",
- "description": "An array of calculated tracking signatures for this tracking item.",
- "minItems": 1,
- "items": {
- "description": "A calculated tracking signature value and metadata.",
- "required": [
- "algorithm",
- "value"
- ],
- "properties": {
- "algorithm": {
- "type": "string",
- "description": "The algorithm used to generate the signature."
- },
- "value": {
- "type": "string",
- "description": "The result of this signature algorithm."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "Each tracking type must declare its own type."
- }
- }
- },
- "flags": {
- "description": "Flags that can be attached to vulnerabilities.",
- "type": "array",
- "items": {
- "type": "object",
- "description": "Informational flags identified and assigned to a vulnerability.",
- "required": [
- "type",
- "origin",
- "description"
- ],
- "properties": {
- "type": {
- "type": "string",
- "minLength": 1,
- "description": "Result of the scan.",
- "enum": [
- "flagged-as-likely-false-positive"
- ]
- },
- "origin": {
- "minLength": 1,
- "description": "Tool that issued the flag.",
- "type": "string"
- },
- "description": {
- "minLength": 1,
- "description": "What the flag is about.",
- "type": "string"
- }
- }
- }
- },
- "location": {
- "type": "object",
- "description": "Identifies the vulnerability's location.",
- "required": [
- "dependency",
- "operating_system",
- "image"
- ],
- "properties": {
- "dependency": {
- "type": "object",
- "description": "Describes the dependency of a project where the vulnerability is located.",
- "properties": {
- "package": {
- "type": "object",
- "description": "Provides information on the package where the vulnerability is located.",
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the package where the vulnerability is located."
- }
- }
- },
- "version": {
- "type": "string",
- "description": "Version of the vulnerable package."
- },
- "iid": {
- "description": "ID that identifies the dependency in the scope of a dependency file.",
- "type": "number"
- },
- "direct": {
- "type": "boolean",
- "description": "Tells whether this is a direct, top-level dependency of the scanned project."
- },
- "dependency_path": {
- "type": "array",
- "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.",
- "items": {
- "type": "object",
- "required": [
- "iid"
- ],
- "properties": {
- "iid": {
- "type": "number",
- "description": "ID that is unique in the scope of a parent object, and specific to the resource type."
- }
- }
- }
- }
- }
- },
- "operating_system": {
- "type": "string",
- "minLength": 1,
- "description": "The operating system that contains the vulnerable package."
- },
- "image": {
- "type": "string",
- "minLength": 1,
- "description": "The analyzed Docker image."
- }
- }
- }
- }
- }
- },
- "remediations": {
- "type": "array",
- "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
- "items": {
- "type": "object",
- "required": [
- "fixes",
- "summary",
- "diff"
- ],
- "properties": {
- "fixes": {
- "type": "array",
- "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
- "items": {
- "type": "object",
- "required": [
- "cve"
- ],
- "properties": {
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- }
- }
- }
- },
- "summary": {
- "type": "string",
- "minLength": 1,
- "description": "An overview of how the vulnerabilities were fixed."
- },
- "diff": {
- "type": "string",
- "minLength": 1,
- "description": "A base64-encoded remediation code diff, compatible with git apply."
- }
- }
- }
- }
- }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.3/coverage-fuzzing-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.3/coverage-fuzzing-report-format.json
deleted file mode 100644
index 20038dcb21c..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.3/coverage-fuzzing-report-format.json
+++ /dev/null
@@ -1,874 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Report format for GitLab Fuzz Testing",
- "description": "This schema provides the report format for Coverage Guided Fuzz Testing (https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing).",
- "definitions": {
- "detail_type": {
- "oneOf": [
- {
- "$ref": "#/definitions/named_list"
- },
- {
- "$ref": "#/definitions/list"
- },
- {
- "$ref": "#/definitions/table"
- },
- {
- "$ref": "#/definitions/text"
- },
- {
- "$ref": "#/definitions/url"
- },
- {
- "$ref": "#/definitions/code"
- },
- {
- "$ref": "#/definitions/value"
- },
- {
- "$ref": "#/definitions/diff"
- },
- {
- "$ref": "#/definitions/markdown"
- },
- {
- "$ref": "#/definitions/commit"
- },
- {
- "$ref": "#/definitions/file_location"
- },
- {
- "$ref": "#/definitions/module_location"
- }
- ]
- },
- "text_value": {
- "type": "string"
- },
- "named_field": {
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "$ref": "#/definitions/text_value",
- "minLength": 1
- },
- "description": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "named_list": {
- "type": "object",
- "description": "An object with named and typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "named-list"
- },
- "items": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- {
- "$ref": "#/definitions/named_field"
- },
- {
- "$ref": "#/definitions/detail_type"
- }
- ]
- }
- }
- }
- }
- },
- "list": {
- "type": "object",
- "description": "A list of typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- },
- "table": {
- "type": "object",
- "description": "A table of typed fields",
- "required": [
- "type",
- "rows"
- ],
- "properties": {
- "type": {
- "const": "table"
- },
- "header": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- },
- "rows": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- }
- },
- "text": {
- "type": "object",
- "description": "Raw text",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "text"
- },
- "value": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "url": {
- "type": "object",
- "description": "A single URL",
- "required": [
- "type",
- "href"
- ],
- "properties": {
- "type": {
- "const": "url"
- },
- "text": {
- "$ref": "#/definitions/text_value"
- },
- "href": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "http://mysite.com"
- ]
- }
- }
- },
- "code": {
- "type": "object",
- "description": "A codeblock",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "code"
- },
- "value": {
- "type": "string"
- },
- "lang": {
- "type": "string",
- "description": "A programming language"
- }
- }
- },
- "value": {
- "type": "object",
- "description": "A field that can store a range of types of value",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "value"
- },
- "value": {
- "type": [
- "number",
- "string",
- "boolean"
- ]
- }
- }
- },
- "diff": {
- "type": "object",
- "description": "A diff",
- "required": [
- "type",
- "before",
- "after"
- ],
- "properties": {
- "type": {
- "const": "diff"
- },
- "before": {
- "type": "string"
- },
- "after": {
- "type": "string"
- }
- }
- },
- "markdown": {
- "type": "object",
- "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "markdown"
- },
- "value": {
- "$ref": "#/definitions/text_value",
- "examples": [
- "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
- ]
- }
- }
- },
- "commit": {
- "type": "object",
- "description": "A commit/tag/branch within the GitLab project",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "commit"
- },
- "value": {
- "type": "string",
- "description": "The commit SHA",
- "minLength": 1
- }
- }
- },
- "file_location": {
- "type": "object",
- "description": "A location within a file in the project",
- "required": [
- "type",
- "file_name",
- "line_start"
- ],
- "properties": {
- "type": {
- "const": "file-location"
- },
- "file_name": {
- "type": "string",
- "minLength": 1
- },
- "line_start": {
- "type": "integer"
- },
- "line_end": {
- "type": "integer"
- }
- }
- },
- "module_location": {
- "type": "object",
- "description": "A location within a binary module of the form module+relative_offset",
- "required": [
- "type",
- "module_name",
- "offset"
- ],
- "properties": {
- "type": {
- "const": "module-location"
- },
- "module_name": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "compiled_binary"
- ]
- },
- "offset": {
- "type": "integer",
- "examples": [
- 100
- ]
- }
- }
- }
- },
- "self": {
- "version": "14.0.3"
- },
- "required": [
- "version",
- "vulnerabilities"
- ],
- "additionalProperties": true,
- "properties": {
- "scan": {
- "type": "object",
- "required": [
- "end_time",
- "scanner",
- "start_time",
- "status",
- "type"
- ],
- "properties": {
- "end_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-01-28T03:26:02"
- ]
- },
- "messages": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Communication intended for the initiator of a scan.",
- "required": [
- "level",
- "value"
- ],
- "properties": {
- "level": {
- "type": "string",
- "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
- "enum": [
- "info",
- "warn",
- "fatal"
- ],
- "examples": [
- "info"
- ]
- },
- "value": {
- "type": "string",
- "description": "The message to communicate.",
- "minLength": 1,
- "examples": [
- "Permission denied, scanning aborted"
- ]
- }
- }
- }
- },
- "analyzer": {
- "type": "object",
- "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the analyzer.",
- "minLength": 1,
- "examples": [
- "gitlab-dast"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the analyzer, not required to be unique.",
- "minLength": 1,
- "examples": [
- "GitLab DAST"
- ]
- },
- "url": {
- "type": "string",
- "format": "uri",
- "pattern": "^https?://.+",
- "description": "A link to more information about the analyzer.",
- "examples": [
- "https://docs.gitlab.com/ee/user/application_security/dast"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the analyzer.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- },
- "version": {
- "type": "string",
- "description": "The version of the analyzer.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- }
- }
- },
- "scanner": {
- "type": "object",
- "description": "Object defining the scanner used to perform the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the scanner.",
- "minLength": 1,
- "examples": [
- "my-sast-scanner"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the scanner, not required to be unique.",
- "minLength": 1,
- "examples": [
- "My SAST Scanner"
- ]
- },
- "url": {
- "type": "string",
- "description": "A link to more information about the scanner.",
- "examples": [
- "https://scanner.url"
- ]
- },
- "version": {
- "type": "string",
- "description": "The version of the scanner.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the scanner.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- }
- }
- },
- "start_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-02-14T16:01:59"
- ]
- },
- "status": {
- "type": "string",
- "description": "Result of the scan.",
- "enum": [
- "success",
- "failure"
- ]
- },
- "type": {
- "type": "string",
- "description": "Type of the scan.",
- "enum": [
- "coverage_fuzzing"
- ]
- }
- }
- },
- "schema": {
- "type": "string",
- "description": "URI pointing to the validating security report schema.",
- "format": "uri"
- },
- "version": {
- "type": "string",
- "description": "The version of the schema to which the JSON report conforms.",
- "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
- },
- "vulnerabilities": {
- "type": "array",
- "description": "Array of vulnerability objects.",
- "items": {
- "type": "object",
- "description": "Describes the vulnerability.",
- "required": [
- "category",
- "cve",
- "identifiers",
- "location",
- "scanner"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
- "examples": [
- "642735a5-1425-428d-8d4e-3c854885a3c9"
- ]
- },
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
- "name": {
- "type": "string",
- "description": "The name of the vulnerability. This must not include the finding's specific information."
- },
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
- "description": {
- "type": "string",
- "description": "A long text section describing the vulnerability more fully."
- },
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
- "severity": {
- "type": "string",
- "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Info",
- "Unknown",
- "Low",
- "Medium",
- "High",
- "Critical"
- ]
- },
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
- "solution": {
- "type": "string",
- "description": "Explanation of how to fix the vulnerability."
- },
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
- "identifiers": {
- "type": "array",
- "minItems": 1,
- "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
- "items": {
- "type": "object",
- "required": [
- "type",
- "name",
- "value"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
- "minLength": 1
- },
- "name": {
- "type": "string",
- "description": "Human-readable name of the identifier.",
- "minLength": 1
- },
- "url": {
- "type": "string",
- "description": "URL of the identifier's documentation.",
- "format": "uri"
- },
- "value": {
- "type": "string",
- "description": "Value of the identifier, for matching purpose.",
- "minLength": 1
- }
- }
- }
- },
- "links": {
- "type": "array",
- "description": "An array of references to external documentation or articles that describe the vulnerability.",
- "items": {
- "type": "object",
- "required": [
- "url"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the vulnerability details link."
- },
- "url": {
- "type": "string",
- "description": "URL of the vulnerability details document.",
- "format": "uri"
- }
- }
- }
- },
- "details": {
- "$ref": "#/definitions/named_list/properties/items"
- },
- "tracking": {
- "description": "Describes how this vulnerability should be tracked as the project changes.",
- "oneOf": [
- {
- "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
- "required": [
- "items"
- ],
- "properties": {
- "type": {
- "const": "source"
- },
- "items": {
- "type": "array",
- "items": {
- "description": "An item that should be tracked using source-specific tracking methods.",
- "type": "object",
- "required": [
- "signatures"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the file that includes the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the file that includes the vulnerability."
- },
- "signatures": {
- "type": "array",
- "description": "An array of calculated tracking signatures for this tracking item.",
- "minItems": 1,
- "items": {
- "description": "A calculated tracking signature value and metadata.",
- "required": [
- "algorithm",
- "value"
- ],
- "properties": {
- "algorithm": {
- "type": "string",
- "description": "The algorithm used to generate the signature."
- },
- "value": {
- "type": "string",
- "description": "The result of this signature algorithm."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "Each tracking type must declare its own type."
- }
- }
- },
- "flags": {
- "description": "Flags that can be attached to vulnerabilities.",
- "type": "array",
- "items": {
- "type": "object",
- "description": "Informational flags identified and assigned to a vulnerability.",
- "required": [
- "type",
- "origin",
- "description"
- ],
- "properties": {
- "type": {
- "type": "string",
- "minLength": 1,
- "description": "Result of the scan.",
- "enum": [
- "flagged-as-likely-false-positive"
- ]
- },
- "origin": {
- "minLength": 1,
- "description": "Tool that issued the flag.",
- "type": "string"
- },
- "description": {
- "minLength": 1,
- "description": "What the flag is about.",
- "type": "string"
- }
- }
- }
- },
- "location": {
- "description": "The location of the error",
- "type": "object",
- "properties": {
- "crash_address": {
- "type": "string",
- "description": "The relative address in memory were the crash occurred.",
- "examples": [
- "0xabababab"
- ]
- },
- "stacktrace_snippet": {
- "type": "string",
- "description": "The stack trace recorded during fuzzing resulting the crash.",
- "examples": [
- "func_a+0xabcd\nfunc_b+0xabcc"
- ]
- },
- "crash_state": {
- "type": "string",
- "description": "Minimised and normalized crash stack-trace (called crash_state).",
- "examples": [
- "func_a+0xa\nfunc_b+0xb\nfunc_c+0xc"
- ]
- },
- "crash_type": {
- "type": "string",
- "description": "Type of the crash.",
- "examples": [
- "Heap-Buffer-overflow",
- "Division-by-zero"
- ]
- }
- }
- }
- }
- }
- },
- "remediations": {
- "type": "array",
- "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
- "items": {
- "type": "object",
- "required": [
- "fixes",
- "summary",
- "diff"
- ],
- "properties": {
- "fixes": {
- "type": "array",
- "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
- "items": {
- "type": "object",
- "required": [
- "cve"
- ],
- "properties": {
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- }
- }
- }
- },
- "summary": {
- "type": "string",
- "minLength": 1,
- "description": "An overview of how the vulnerabilities were fixed."
- },
- "diff": {
- "type": "string",
- "minLength": 1,
- "description": "A base64-encoded remediation code diff, compatible with git apply."
- }
- }
- }
- }
- }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.3/dast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.3/dast-report-format.json
deleted file mode 100644
index 37b98a73233..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.3/dast-report-format.json
+++ /dev/null
@@ -1,1291 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Report format for GitLab DAST",
- "description": "This schema provides the the report format for Dynamic Application Security Testing (https://docs.gitlab.com/ee/user/application_security/dast).",
- "definitions": {
- "detail_type": {
- "oneOf": [
- {
- "$ref": "#/definitions/named_list"
- },
- {
- "$ref": "#/definitions/list"
- },
- {
- "$ref": "#/definitions/table"
- },
- {
- "$ref": "#/definitions/text"
- },
- {
- "$ref": "#/definitions/url"
- },
- {
- "$ref": "#/definitions/code"
- },
- {
- "$ref": "#/definitions/value"
- },
- {
- "$ref": "#/definitions/diff"
- },
- {
- "$ref": "#/definitions/markdown"
- },
- {
- "$ref": "#/definitions/commit"
- },
- {
- "$ref": "#/definitions/file_location"
- },
- {
- "$ref": "#/definitions/module_location"
- }
- ]
- },
- "text_value": {
- "type": "string"
- },
- "named_field": {
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "$ref": "#/definitions/text_value",
- "minLength": 1
- },
- "description": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "named_list": {
- "type": "object",
- "description": "An object with named and typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "named-list"
- },
- "items": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- {
- "$ref": "#/definitions/named_field"
- },
- {
- "$ref": "#/definitions/detail_type"
- }
- ]
- }
- }
- }
- }
- },
- "list": {
- "type": "object",
- "description": "A list of typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- },
- "table": {
- "type": "object",
- "description": "A table of typed fields",
- "required": [
- "type",
- "rows"
- ],
- "properties": {
- "type": {
- "const": "table"
- },
- "header": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- },
- "rows": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- }
- },
- "text": {
- "type": "object",
- "description": "Raw text",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "text"
- },
- "value": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "url": {
- "type": "object",
- "description": "A single URL",
- "required": [
- "type",
- "href"
- ],
- "properties": {
- "type": {
- "const": "url"
- },
- "text": {
- "$ref": "#/definitions/text_value"
- },
- "href": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "http://mysite.com"
- ]
- }
- }
- },
- "code": {
- "type": "object",
- "description": "A codeblock",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "code"
- },
- "value": {
- "type": "string"
- },
- "lang": {
- "type": "string",
- "description": "A programming language"
- }
- }
- },
- "value": {
- "type": "object",
- "description": "A field that can store a range of types of value",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "value"
- },
- "value": {
- "type": [
- "number",
- "string",
- "boolean"
- ]
- }
- }
- },
- "diff": {
- "type": "object",
- "description": "A diff",
- "required": [
- "type",
- "before",
- "after"
- ],
- "properties": {
- "type": {
- "const": "diff"
- },
- "before": {
- "type": "string"
- },
- "after": {
- "type": "string"
- }
- }
- },
- "markdown": {
- "type": "object",
- "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "markdown"
- },
- "value": {
- "$ref": "#/definitions/text_value",
- "examples": [
- "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
- ]
- }
- }
- },
- "commit": {
- "type": "object",
- "description": "A commit/tag/branch within the GitLab project",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "commit"
- },
- "value": {
- "type": "string",
- "description": "The commit SHA",
- "minLength": 1
- }
- }
- },
- "file_location": {
- "type": "object",
- "description": "A location within a file in the project",
- "required": [
- "type",
- "file_name",
- "line_start"
- ],
- "properties": {
- "type": {
- "const": "file-location"
- },
- "file_name": {
- "type": "string",
- "minLength": 1
- },
- "line_start": {
- "type": "integer"
- },
- "line_end": {
- "type": "integer"
- }
- }
- },
- "module_location": {
- "type": "object",
- "description": "A location within a binary module of the form module+relative_offset",
- "required": [
- "type",
- "module_name",
- "offset"
- ],
- "properties": {
- "type": {
- "const": "module-location"
- },
- "module_name": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "compiled_binary"
- ]
- },
- "offset": {
- "type": "integer",
- "examples": [
- 100
- ]
- }
- }
- }
- },
- "self": {
- "version": "14.0.3"
- },
- "required": [
- "version",
- "vulnerabilities"
- ],
- "additionalProperties": true,
- "properties": {
- "scan": {
- "type": "object",
- "required": [
- "end_time",
- "scanned_resources",
- "scanner",
- "start_time",
- "status",
- "type"
- ],
- "properties": {
- "end_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-01-28T03:26:02"
- ]
- },
- "messages": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Communication intended for the initiator of a scan.",
- "required": [
- "level",
- "value"
- ],
- "properties": {
- "level": {
- "type": "string",
- "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
- "enum": [
- "info",
- "warn",
- "fatal"
- ],
- "examples": [
- "info"
- ]
- },
- "value": {
- "type": "string",
- "description": "The message to communicate.",
- "minLength": 1,
- "examples": [
- "Permission denied, scanning aborted"
- ]
- }
- }
- }
- },
- "analyzer": {
- "type": "object",
- "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the analyzer.",
- "minLength": 1,
- "examples": [
- "gitlab-dast"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the analyzer, not required to be unique.",
- "minLength": 1,
- "examples": [
- "GitLab DAST"
- ]
- },
- "url": {
- "type": "string",
- "format": "uri",
- "pattern": "^https?://.+",
- "description": "A link to more information about the analyzer.",
- "examples": [
- "https://docs.gitlab.com/ee/user/application_security/dast"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the analyzer.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- },
- "version": {
- "type": "string",
- "description": "The version of the analyzer.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- }
- }
- },
- "scanner": {
- "type": "object",
- "description": "Object defining the scanner used to perform the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the scanner.",
- "minLength": 1,
- "examples": [
- "my-sast-scanner"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the scanner, not required to be unique.",
- "minLength": 1,
- "examples": [
- "My SAST Scanner"
- ]
- },
- "url": {
- "type": "string",
- "description": "A link to more information about the scanner.",
- "examples": [
- "https://scanner.url"
- ]
- },
- "version": {
- "type": "string",
- "description": "The version of the scanner.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the scanner.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- }
- }
- },
- "start_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-02-14T16:01:59"
- ]
- },
- "status": {
- "type": "string",
- "description": "Result of the scan.",
- "enum": [
- "success",
- "failure"
- ]
- },
- "type": {
- "type": "string",
- "description": "Type of the scan.",
- "enum": [
- "dast",
- "api_fuzzing"
- ]
- },
- "scanned_resources": {
- "type": "array",
- "description": "The attack surface scanned by DAST.",
- "items": {
- "type": "object",
- "required": [
- "method",
- "url",
- "type"
- ],
- "properties": {
- "method": {
- "type": "string",
- "minLength": 1,
- "description": "HTTP method of the scanned resource.",
- "examples": [
- "GET",
- "POST",
- "HEAD"
- ]
- },
- "url": {
- "type": "string",
- "minLength": 1,
- "description": "URL of the scanned resource.",
- "examples": [
- "http://my.site.com/a-page"
- ]
- },
- "type": {
- "type": "string",
- "minLength": 1,
- "description": "Type of the scanned resource, for DAST, this must be 'url'.",
- "examples": [
- "url"
- ]
- }
- }
- }
- }
- }
- },
- "schema": {
- "type": "string",
- "description": "URI pointing to the validating security report schema.",
- "format": "uri"
- },
- "version": {
- "type": "string",
- "description": "The version of the schema to which the JSON report conforms.",
- "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
- },
- "vulnerabilities": {
- "type": "array",
- "description": "Array of vulnerability objects.",
- "items": {
- "type": "object",
- "description": "Describes the vulnerability.",
- "required": [
- "category",
- "cve",
- "identifiers",
- "location",
- "scanner"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
- "examples": [
- "642735a5-1425-428d-8d4e-3c854885a3c9"
- ]
- },
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
- "name": {
- "type": "string",
- "description": "The name of the vulnerability. This must not include the finding's specific information."
- },
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
- "description": {
- "type": "string",
- "description": "A long text section describing the vulnerability more fully."
- },
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
- "severity": {
- "type": "string",
- "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Info",
- "Unknown",
- "Low",
- "Medium",
- "High",
- "Critical"
- ]
- },
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
- "solution": {
- "type": "string",
- "description": "Explanation of how to fix the vulnerability."
- },
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
- "identifiers": {
- "type": "array",
- "minItems": 1,
- "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
- "items": {
- "type": "object",
- "required": [
- "type",
- "name",
- "value"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
- "minLength": 1
- },
- "name": {
- "type": "string",
- "description": "Human-readable name of the identifier.",
- "minLength": 1
- },
- "url": {
- "type": "string",
- "description": "URL of the identifier's documentation.",
- "format": "uri"
- },
- "value": {
- "type": "string",
- "description": "Value of the identifier, for matching purpose.",
- "minLength": 1
- }
- }
- }
- },
- "links": {
- "type": "array",
- "description": "An array of references to external documentation or articles that describe the vulnerability.",
- "items": {
- "type": "object",
- "required": [
- "url"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the vulnerability details link."
- },
- "url": {
- "type": "string",
- "description": "URL of the vulnerability details document.",
- "format": "uri"
- }
- }
- }
- },
- "details": {
- "$ref": "#/definitions/named_list/properties/items"
- },
- "tracking": {
- "description": "Describes how this vulnerability should be tracked as the project changes.",
- "oneOf": [
- {
- "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
- "required": [
- "items"
- ],
- "properties": {
- "type": {
- "const": "source"
- },
- "items": {
- "type": "array",
- "items": {
- "description": "An item that should be tracked using source-specific tracking methods.",
- "type": "object",
- "required": [
- "signatures"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the file that includes the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the file that includes the vulnerability."
- },
- "signatures": {
- "type": "array",
- "description": "An array of calculated tracking signatures for this tracking item.",
- "minItems": 1,
- "items": {
- "description": "A calculated tracking signature value and metadata.",
- "required": [
- "algorithm",
- "value"
- ],
- "properties": {
- "algorithm": {
- "type": "string",
- "description": "The algorithm used to generate the signature."
- },
- "value": {
- "type": "string",
- "description": "The result of this signature algorithm."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "Each tracking type must declare its own type."
- }
- }
- },
- "flags": {
- "description": "Flags that can be attached to vulnerabilities.",
- "type": "array",
- "items": {
- "type": "object",
- "description": "Informational flags identified and assigned to a vulnerability.",
- "required": [
- "type",
- "origin",
- "description"
- ],
- "properties": {
- "type": {
- "type": "string",
- "minLength": 1,
- "description": "Result of the scan.",
- "enum": [
- "flagged-as-likely-false-positive"
- ]
- },
- "origin": {
- "minLength": 1,
- "description": "Tool that issued the flag.",
- "type": "string"
- },
- "description": {
- "minLength": 1,
- "description": "What the flag is about.",
- "type": "string"
- }
- }
- }
- },
- "evidence": {
- "type": "object",
- "properties": {
- "source": {
- "type": "object",
- "description": "Source of evidence",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "Unique source identifier",
- "examples": [
- "assert:LogAnalysis",
- "assert:StatusCode"
- ]
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Source display name",
- "examples": [
- "Log Analysis",
- "Status Code"
- ]
- },
- "url": {
- "type": "string",
- "description": "Link to additional information",
- "examples": [
- "https://docs.gitlab.com/ee/development/integrations/secure.html"
- ]
- }
- }
- },
- "summary": {
- "type": "string",
- "description": "Human readable string containing evidence of the vulnerability.",
- "examples": [
- "Credit card 4111111111111111 found",
- "Server leaked information nginx/1.17.6"
- ]
- },
- "request": {
- "type": "object",
- "description": "An HTTP request.",
- "required": [
- "headers",
- "method",
- "url"
- ],
- "properties": {
- "headers": {
- "type": "array",
- "description": "HTTP headers present on the request.",
- "items": {
- "type": "object",
- "required": [
- "name",
- "value"
- ],
- "properties": {
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Name of the HTTP header.",
- "examples": [
- "Accept",
- "Content-Length",
- "Content-Type"
- ]
- },
- "value": {
- "type": "string",
- "minLength": 1,
- "description": "Value of the HTTP header.",
- "examples": [
- "*/*",
- "560",
- "application/json; charset=utf-8"
- ]
- }
- }
- }
- },
- "method": {
- "type": "string",
- "minLength": 1,
- "description": "HTTP method used in the request.",
- "examples": [
- "GET",
- "POST"
- ]
- },
- "url": {
- "type": "string",
- "minLength": 1,
- "description": "URL of the request.",
- "examples": [
- "http://my.site.com/vulnerable-endpoint?show-credit-card"
- ]
- },
- "body": {
- "type": "string",
- "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.",
- "examples": [
- "user=jsmith&first=%27&last=smith"
- ]
- }
- }
- },
- "response": {
- "type": "object",
- "description": "An HTTP response.",
- "required": [
- "headers",
- "reason_phrase",
- "status_code"
- ],
- "properties": {
- "headers": {
- "type": "array",
- "description": "HTTP headers present on the request.",
- "items": {
- "type": "object",
- "required": [
- "name",
- "value"
- ],
- "properties": {
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Name of the HTTP header.",
- "examples": [
- "Accept",
- "Content-Length",
- "Content-Type"
- ]
- },
- "value": {
- "type": "string",
- "minLength": 1,
- "description": "Value of the HTTP header.",
- "examples": [
- "*/*",
- "560",
- "application/json; charset=utf-8"
- ]
- }
- }
- }
- },
- "reason_phrase": {
- "type": "string",
- "description": "HTTP reason phrase of the response.",
- "examples": [
- "OK",
- "Internal Server Error"
- ]
- },
- "status_code": {
- "type": "integer",
- "description": "HTTP status code of the response.",
- "examples": [
- 200,
- 500
- ]
- },
- "body": {
- "type": "string",
- "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.",
- "examples": [
- "{\"user_id\": 2}"
- ]
- }
- }
- },
- "supporting_messages": {
- "type": "array",
- "description": "Array of supporting http messages.",
- "items": {
- "type": "object",
- "description": "A supporting http message.",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Message display name.",
- "examples": [
- "Unmodified",
- "Recorded"
- ]
- },
- "request": {
- "type": "object",
- "description": "An HTTP request.",
- "required": [
- "headers",
- "method",
- "url"
- ],
- "properties": {
- "headers": {
- "type": "array",
- "description": "HTTP headers present on the request.",
- "items": {
- "type": "object",
- "required": [
- "name",
- "value"
- ],
- "properties": {
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Name of the HTTP header.",
- "examples": [
- "Accept",
- "Content-Length",
- "Content-Type"
- ]
- },
- "value": {
- "type": "string",
- "minLength": 1,
- "description": "Value of the HTTP header.",
- "examples": [
- "*/*",
- "560",
- "application/json; charset=utf-8"
- ]
- }
- }
- }
- },
- "method": {
- "type": "string",
- "minLength": 1,
- "description": "HTTP method used in the request.",
- "examples": [
- "GET",
- "POST"
- ]
- },
- "url": {
- "type": "string",
- "minLength": 1,
- "description": "URL of the request.",
- "examples": [
- "http://my.site.com/vulnerable-endpoint?show-credit-card"
- ]
- },
- "body": {
- "type": "string",
- "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.",
- "examples": [
- "user=jsmith&first=%27&last=smith"
- ]
- }
- }
- },
- "response": {
- "type": "object",
- "description": "An HTTP response.",
- "required": [
- "headers",
- "reason_phrase",
- "status_code"
- ],
- "properties": {
- "headers": {
- "type": "array",
- "description": "HTTP headers present on the request.",
- "items": {
- "type": "object",
- "required": [
- "name",
- "value"
- ],
- "properties": {
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Name of the HTTP header.",
- "examples": [
- "Accept",
- "Content-Length",
- "Content-Type"
- ]
- },
- "value": {
- "type": "string",
- "minLength": 1,
- "description": "Value of the HTTP header.",
- "examples": [
- "*/*",
- "560",
- "application/json; charset=utf-8"
- ]
- }
- }
- }
- },
- "reason_phrase": {
- "type": "string",
- "description": "HTTP reason phrase of the response.",
- "examples": [
- "OK",
- "Internal Server Error"
- ]
- },
- "status_code": {
- "type": "integer",
- "description": "HTTP status code of the response.",
- "examples": [
- 200,
- 500
- ]
- },
- "body": {
- "type": "string",
- "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.",
- "examples": [
- "{\"user_id\": 2}"
- ]
- }
- }
- }
- }
- }
- }
- }
- },
- "location": {
- "type": "object",
- "description": "Identifies the vulnerability's location.",
- "properties": {
- "hostname": {
- "type": "string",
- "description": "The protocol, domain, and port of the application where the vulnerability was found."
- },
- "method": {
- "type": "string",
- "description": "The HTTP method that was used to request the URL where the vulnerability was found."
- },
- "param": {
- "type": "string",
- "description": "A value provided by a vulnerability rule related to the found vulnerability. Examples include a header value, or a parameter used in a HTTP POST."
- },
- "path": {
- "type": "string",
- "description": "The path of the URL where the vulnerability was found. Typically, this would start with a forward slash."
- }
- }
- },
- "assets": {
- "type": "array",
- "description": "Array of build assets associated with vulnerability.",
- "items": {
- "type": "object",
- "description": "Describes an asset associated with vulnerability.",
- "required": [
- "type",
- "name",
- "url"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "The type of asset",
- "enum": [
- "http_session",
- "postman"
- ]
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Display name for asset",
- "examples": [
- "HTTP Messages",
- "Postman Collection"
- ]
- },
- "url": {
- "type": "string",
- "minLength": 1,
- "description": "Link to asset in build artifacts",
- "examples": [
- "https://gitlab.com/gitlab-org/security-products/dast/-/jobs/626397001/artifacts/file//output/zap_session.data"
- ]
- }
- }
- }
- },
- "discovered_at": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss.sss, representing when the vulnerability was discovered",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}\\.\\d{3}$",
- "examples": [
- "2020-01-28T03:26:02.956"
- ]
- }
- }
- }
- },
- "remediations": {
- "type": "array",
- "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
- "items": {
- "type": "object",
- "required": [
- "fixes",
- "summary",
- "diff"
- ],
- "properties": {
- "fixes": {
- "type": "array",
- "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
- "items": {
- "type": "object",
- "required": [
- "cve"
- ],
- "properties": {
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- }
- }
- }
- },
- "summary": {
- "type": "string",
- "minLength": 1,
- "description": "An overview of how the vulnerabilities were fixed."
- },
- "diff": {
- "type": "string",
- "minLength": 1,
- "description": "A base64-encoded remediation code diff, compatible with git apply."
- }
- }
- }
- }
- }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.3/dependency-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.3/dependency-scanning-report-format.json
deleted file mode 100644
index 5e9bbeec1a9..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.3/dependency-scanning-report-format.json
+++ /dev/null
@@ -1,968 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Report format for GitLab Dependency Scanning",
- "description": "This schema provides the the report format for Dependency Scanning analyzers (https://docs.gitlab.com/ee/user/application_security/dependency_scanning).",
- "definitions": {
- "detail_type": {
- "oneOf": [
- {
- "$ref": "#/definitions/named_list"
- },
- {
- "$ref": "#/definitions/list"
- },
- {
- "$ref": "#/definitions/table"
- },
- {
- "$ref": "#/definitions/text"
- },
- {
- "$ref": "#/definitions/url"
- },
- {
- "$ref": "#/definitions/code"
- },
- {
- "$ref": "#/definitions/value"
- },
- {
- "$ref": "#/definitions/diff"
- },
- {
- "$ref": "#/definitions/markdown"
- },
- {
- "$ref": "#/definitions/commit"
- },
- {
- "$ref": "#/definitions/file_location"
- },
- {
- "$ref": "#/definitions/module_location"
- }
- ]
- },
- "text_value": {
- "type": "string"
- },
- "named_field": {
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "$ref": "#/definitions/text_value",
- "minLength": 1
- },
- "description": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "named_list": {
- "type": "object",
- "description": "An object with named and typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "named-list"
- },
- "items": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- {
- "$ref": "#/definitions/named_field"
- },
- {
- "$ref": "#/definitions/detail_type"
- }
- ]
- }
- }
- }
- }
- },
- "list": {
- "type": "object",
- "description": "A list of typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- },
- "table": {
- "type": "object",
- "description": "A table of typed fields",
- "required": [
- "type",
- "rows"
- ],
- "properties": {
- "type": {
- "const": "table"
- },
- "header": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- },
- "rows": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- }
- },
- "text": {
- "type": "object",
- "description": "Raw text",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "text"
- },
- "value": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "url": {
- "type": "object",
- "description": "A single URL",
- "required": [
- "type",
- "href"
- ],
- "properties": {
- "type": {
- "const": "url"
- },
- "text": {
- "$ref": "#/definitions/text_value"
- },
- "href": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "http://mysite.com"
- ]
- }
- }
- },
- "code": {
- "type": "object",
- "description": "A codeblock",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "code"
- },
- "value": {
- "type": "string"
- },
- "lang": {
- "type": "string",
- "description": "A programming language"
- }
- }
- },
- "value": {
- "type": "object",
- "description": "A field that can store a range of types of value",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "value"
- },
- "value": {
- "type": [
- "number",
- "string",
- "boolean"
- ]
- }
- }
- },
- "diff": {
- "type": "object",
- "description": "A diff",
- "required": [
- "type",
- "before",
- "after"
- ],
- "properties": {
- "type": {
- "const": "diff"
- },
- "before": {
- "type": "string"
- },
- "after": {
- "type": "string"
- }
- }
- },
- "markdown": {
- "type": "object",
- "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "markdown"
- },
- "value": {
- "$ref": "#/definitions/text_value",
- "examples": [
- "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
- ]
- }
- }
- },
- "commit": {
- "type": "object",
- "description": "A commit/tag/branch within the GitLab project",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "commit"
- },
- "value": {
- "type": "string",
- "description": "The commit SHA",
- "minLength": 1
- }
- }
- },
- "file_location": {
- "type": "object",
- "description": "A location within a file in the project",
- "required": [
- "type",
- "file_name",
- "line_start"
- ],
- "properties": {
- "type": {
- "const": "file-location"
- },
- "file_name": {
- "type": "string",
- "minLength": 1
- },
- "line_start": {
- "type": "integer"
- },
- "line_end": {
- "type": "integer"
- }
- }
- },
- "module_location": {
- "type": "object",
- "description": "A location within a binary module of the form module+relative_offset",
- "required": [
- "type",
- "module_name",
- "offset"
- ],
- "properties": {
- "type": {
- "const": "module-location"
- },
- "module_name": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "compiled_binary"
- ]
- },
- "offset": {
- "type": "integer",
- "examples": [
- 100
- ]
- }
- }
- }
- },
- "self": {
- "version": "14.0.3"
- },
- "required": [
- "dependency_files",
- "version",
- "vulnerabilities"
- ],
- "additionalProperties": true,
- "properties": {
- "scan": {
- "type": "object",
- "required": [
- "end_time",
- "scanner",
- "start_time",
- "status",
- "type"
- ],
- "properties": {
- "end_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-01-28T03:26:02"
- ]
- },
- "messages": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Communication intended for the initiator of a scan.",
- "required": [
- "level",
- "value"
- ],
- "properties": {
- "level": {
- "type": "string",
- "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
- "enum": [
- "info",
- "warn",
- "fatal"
- ],
- "examples": [
- "info"
- ]
- },
- "value": {
- "type": "string",
- "description": "The message to communicate.",
- "minLength": 1,
- "examples": [
- "Permission denied, scanning aborted"
- ]
- }
- }
- }
- },
- "analyzer": {
- "type": "object",
- "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the analyzer.",
- "minLength": 1,
- "examples": [
- "gitlab-dast"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the analyzer, not required to be unique.",
- "minLength": 1,
- "examples": [
- "GitLab DAST"
- ]
- },
- "url": {
- "type": "string",
- "format": "uri",
- "pattern": "^https?://.+",
- "description": "A link to more information about the analyzer.",
- "examples": [
- "https://docs.gitlab.com/ee/user/application_security/dast"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the analyzer.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- },
- "version": {
- "type": "string",
- "description": "The version of the analyzer.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- }
- }
- },
- "scanner": {
- "type": "object",
- "description": "Object defining the scanner used to perform the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the scanner.",
- "minLength": 1,
- "examples": [
- "my-sast-scanner"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the scanner, not required to be unique.",
- "minLength": 1,
- "examples": [
- "My SAST Scanner"
- ]
- },
- "url": {
- "type": "string",
- "description": "A link to more information about the scanner.",
- "examples": [
- "https://scanner.url"
- ]
- },
- "version": {
- "type": "string",
- "description": "The version of the scanner.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the scanner.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- }
- }
- },
- "start_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-02-14T16:01:59"
- ]
- },
- "status": {
- "type": "string",
- "description": "Result of the scan.",
- "enum": [
- "success",
- "failure"
- ]
- },
- "type": {
- "type": "string",
- "description": "Type of the scan.",
- "enum": [
- "dependency_scanning"
- ]
- }
- }
- },
- "schema": {
- "type": "string",
- "description": "URI pointing to the validating security report schema.",
- "format": "uri"
- },
- "version": {
- "type": "string",
- "description": "The version of the schema to which the JSON report conforms.",
- "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
- },
- "vulnerabilities": {
- "type": "array",
- "description": "Array of vulnerability objects.",
- "items": {
- "type": "object",
- "description": "Describes the vulnerability.",
- "required": [
- "category",
- "cve",
- "identifiers",
- "location",
- "scanner"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
- "examples": [
- "642735a5-1425-428d-8d4e-3c854885a3c9"
- ]
- },
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
- "name": {
- "type": "string",
- "description": "The name of the vulnerability. This must not include the finding's specific information."
- },
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
- "description": {
- "type": "string",
- "description": "A long text section describing the vulnerability more fully."
- },
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
- "severity": {
- "type": "string",
- "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Info",
- "Unknown",
- "Low",
- "Medium",
- "High",
- "Critical"
- ]
- },
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
- "solution": {
- "type": "string",
- "description": "Explanation of how to fix the vulnerability."
- },
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
- "identifiers": {
- "type": "array",
- "minItems": 1,
- "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
- "items": {
- "type": "object",
- "required": [
- "type",
- "name",
- "value"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
- "minLength": 1
- },
- "name": {
- "type": "string",
- "description": "Human-readable name of the identifier.",
- "minLength": 1
- },
- "url": {
- "type": "string",
- "description": "URL of the identifier's documentation.",
- "format": "uri"
- },
- "value": {
- "type": "string",
- "description": "Value of the identifier, for matching purpose.",
- "minLength": 1
- }
- }
- }
- },
- "links": {
- "type": "array",
- "description": "An array of references to external documentation or articles that describe the vulnerability.",
- "items": {
- "type": "object",
- "required": [
- "url"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the vulnerability details link."
- },
- "url": {
- "type": "string",
- "description": "URL of the vulnerability details document.",
- "format": "uri"
- }
- }
- }
- },
- "details": {
- "$ref": "#/definitions/named_list/properties/items"
- },
- "tracking": {
- "description": "Describes how this vulnerability should be tracked as the project changes.",
- "oneOf": [
- {
- "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
- "required": [
- "items"
- ],
- "properties": {
- "type": {
- "const": "source"
- },
- "items": {
- "type": "array",
- "items": {
- "description": "An item that should be tracked using source-specific tracking methods.",
- "type": "object",
- "required": [
- "signatures"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the file that includes the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the file that includes the vulnerability."
- },
- "signatures": {
- "type": "array",
- "description": "An array of calculated tracking signatures for this tracking item.",
- "minItems": 1,
- "items": {
- "description": "A calculated tracking signature value and metadata.",
- "required": [
- "algorithm",
- "value"
- ],
- "properties": {
- "algorithm": {
- "type": "string",
- "description": "The algorithm used to generate the signature."
- },
- "value": {
- "type": "string",
- "description": "The result of this signature algorithm."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "Each tracking type must declare its own type."
- }
- }
- },
- "flags": {
- "description": "Flags that can be attached to vulnerabilities.",
- "type": "array",
- "items": {
- "type": "object",
- "description": "Informational flags identified and assigned to a vulnerability.",
- "required": [
- "type",
- "origin",
- "description"
- ],
- "properties": {
- "type": {
- "type": "string",
- "minLength": 1,
- "description": "Result of the scan.",
- "enum": [
- "flagged-as-likely-false-positive"
- ]
- },
- "origin": {
- "minLength": 1,
- "description": "Tool that issued the flag.",
- "type": "string"
- },
- "description": {
- "minLength": 1,
- "description": "What the flag is about.",
- "type": "string"
- }
- }
- }
- },
- "location": {
- "type": "object",
- "description": "Identifies the vulnerability's location.",
- "required": [
- "file",
- "dependency"
- ],
- "properties": {
- "file": {
- "type": "string",
- "minLength": 1,
- "description": "Path to the manifest or lock file where the dependency is declared (such as yarn.lock)."
- },
- "dependency": {
- "type": "object",
- "description": "Describes the dependency of a project where the vulnerability is located.",
- "properties": {
- "package": {
- "type": "object",
- "description": "Provides information on the package where the vulnerability is located.",
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the package where the vulnerability is located."
- }
- }
- },
- "version": {
- "type": "string",
- "description": "Version of the vulnerable package."
- },
- "iid": {
- "description": "ID that identifies the dependency in the scope of a dependency file.",
- "type": "number"
- },
- "direct": {
- "type": "boolean",
- "description": "Tells whether this is a direct, top-level dependency of the scanned project."
- },
- "dependency_path": {
- "type": "array",
- "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.",
- "items": {
- "type": "object",
- "required": [
- "iid"
- ],
- "properties": {
- "iid": {
- "type": "number",
- "description": "ID that is unique in the scope of a parent object, and specific to the resource type."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- }
- },
- "remediations": {
- "type": "array",
- "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
- "items": {
- "type": "object",
- "required": [
- "fixes",
- "summary",
- "diff"
- ],
- "properties": {
- "fixes": {
- "type": "array",
- "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
- "items": {
- "type": "object",
- "required": [
- "cve"
- ],
- "properties": {
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- }
- }
- }
- },
- "summary": {
- "type": "string",
- "minLength": 1,
- "description": "An overview of how the vulnerabilities were fixed."
- },
- "diff": {
- "type": "string",
- "minLength": 1,
- "description": "A base64-encoded remediation code diff, compatible with git apply."
- }
- }
- }
- },
- "dependency_files": {
- "type": "array",
- "description": "List of dependency files identified in the project.",
- "items": {
- "type": "object",
- "required": [
- "path",
- "package_manager",
- "dependencies"
- ],
- "properties": {
- "path": {
- "type": "string",
- "minLength": 1
- },
- "package_manager": {
- "type": "string",
- "minLength": 1
- },
- "dependencies": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Describes the dependency of a project where the vulnerability is located.",
- "properties": {
- "package": {
- "type": "object",
- "description": "Provides information on the package where the vulnerability is located.",
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the package where the vulnerability is located."
- }
- }
- },
- "version": {
- "type": "string",
- "description": "Version of the vulnerable package."
- },
- "iid": {
- "description": "ID that identifies the dependency in the scope of a dependency file.",
- "type": "number"
- },
- "direct": {
- "type": "boolean",
- "description": "Tells whether this is a direct, top-level dependency of the scanned project."
- },
- "dependency_path": {
- "type": "array",
- "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.",
- "items": {
- "type": "object",
- "required": [
- "iid"
- ],
- "properties": {
- "iid": {
- "type": "number",
- "description": "ID that is unique in the scope of a parent object, and specific to the resource type."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- }
- }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.3/sast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.3/sast-report-format.json
deleted file mode 100644
index 8aa98646818..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.3/sast-report-format.json
+++ /dev/null
@@ -1,869 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Report format for GitLab SAST",
- "description": "This schema provides the report format for Static Application Security Testing analyzers (https://docs.gitlab.com/ee/user/application_security/sast).",
- "definitions": {
- "detail_type": {
- "oneOf": [
- {
- "$ref": "#/definitions/named_list"
- },
- {
- "$ref": "#/definitions/list"
- },
- {
- "$ref": "#/definitions/table"
- },
- {
- "$ref": "#/definitions/text"
- },
- {
- "$ref": "#/definitions/url"
- },
- {
- "$ref": "#/definitions/code"
- },
- {
- "$ref": "#/definitions/value"
- },
- {
- "$ref": "#/definitions/diff"
- },
- {
- "$ref": "#/definitions/markdown"
- },
- {
- "$ref": "#/definitions/commit"
- },
- {
- "$ref": "#/definitions/file_location"
- },
- {
- "$ref": "#/definitions/module_location"
- }
- ]
- },
- "text_value": {
- "type": "string"
- },
- "named_field": {
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "$ref": "#/definitions/text_value",
- "minLength": 1
- },
- "description": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "named_list": {
- "type": "object",
- "description": "An object with named and typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "named-list"
- },
- "items": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- {
- "$ref": "#/definitions/named_field"
- },
- {
- "$ref": "#/definitions/detail_type"
- }
- ]
- }
- }
- }
- }
- },
- "list": {
- "type": "object",
- "description": "A list of typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- },
- "table": {
- "type": "object",
- "description": "A table of typed fields",
- "required": [
- "type",
- "rows"
- ],
- "properties": {
- "type": {
- "const": "table"
- },
- "header": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- },
- "rows": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- }
- },
- "text": {
- "type": "object",
- "description": "Raw text",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "text"
- },
- "value": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "url": {
- "type": "object",
- "description": "A single URL",
- "required": [
- "type",
- "href"
- ],
- "properties": {
- "type": {
- "const": "url"
- },
- "text": {
- "$ref": "#/definitions/text_value"
- },
- "href": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "http://mysite.com"
- ]
- }
- }
- },
- "code": {
- "type": "object",
- "description": "A codeblock",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "code"
- },
- "value": {
- "type": "string"
- },
- "lang": {
- "type": "string",
- "description": "A programming language"
- }
- }
- },
- "value": {
- "type": "object",
- "description": "A field that can store a range of types of value",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "value"
- },
- "value": {
- "type": [
- "number",
- "string",
- "boolean"
- ]
- }
- }
- },
- "diff": {
- "type": "object",
- "description": "A diff",
- "required": [
- "type",
- "before",
- "after"
- ],
- "properties": {
- "type": {
- "const": "diff"
- },
- "before": {
- "type": "string"
- },
- "after": {
- "type": "string"
- }
- }
- },
- "markdown": {
- "type": "object",
- "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "markdown"
- },
- "value": {
- "$ref": "#/definitions/text_value",
- "examples": [
- "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
- ]
- }
- }
- },
- "commit": {
- "type": "object",
- "description": "A commit/tag/branch within the GitLab project",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "commit"
- },
- "value": {
- "type": "string",
- "description": "The commit SHA",
- "minLength": 1
- }
- }
- },
- "file_location": {
- "type": "object",
- "description": "A location within a file in the project",
- "required": [
- "type",
- "file_name",
- "line_start"
- ],
- "properties": {
- "type": {
- "const": "file-location"
- },
- "file_name": {
- "type": "string",
- "minLength": 1
- },
- "line_start": {
- "type": "integer"
- },
- "line_end": {
- "type": "integer"
- }
- }
- },
- "module_location": {
- "type": "object",
- "description": "A location within a binary module of the form module+relative_offset",
- "required": [
- "type",
- "module_name",
- "offset"
- ],
- "properties": {
- "type": {
- "const": "module-location"
- },
- "module_name": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "compiled_binary"
- ]
- },
- "offset": {
- "type": "integer",
- "examples": [
- 100
- ]
- }
- }
- }
- },
- "self": {
- "version": "14.0.3"
- },
- "required": [
- "version",
- "vulnerabilities"
- ],
- "additionalProperties": true,
- "properties": {
- "scan": {
- "type": "object",
- "required": [
- "end_time",
- "scanner",
- "start_time",
- "status",
- "type"
- ],
- "properties": {
- "end_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-01-28T03:26:02"
- ]
- },
- "messages": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Communication intended for the initiator of a scan.",
- "required": [
- "level",
- "value"
- ],
- "properties": {
- "level": {
- "type": "string",
- "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
- "enum": [
- "info",
- "warn",
- "fatal"
- ],
- "examples": [
- "info"
- ]
- },
- "value": {
- "type": "string",
- "description": "The message to communicate.",
- "minLength": 1,
- "examples": [
- "Permission denied, scanning aborted"
- ]
- }
- }
- }
- },
- "analyzer": {
- "type": "object",
- "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the analyzer.",
- "minLength": 1,
- "examples": [
- "gitlab-dast"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the analyzer, not required to be unique.",
- "minLength": 1,
- "examples": [
- "GitLab DAST"
- ]
- },
- "url": {
- "type": "string",
- "format": "uri",
- "pattern": "^https?://.+",
- "description": "A link to more information about the analyzer.",
- "examples": [
- "https://docs.gitlab.com/ee/user/application_security/dast"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the analyzer.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- },
- "version": {
- "type": "string",
- "description": "The version of the analyzer.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- }
- }
- },
- "scanner": {
- "type": "object",
- "description": "Object defining the scanner used to perform the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the scanner.",
- "minLength": 1,
- "examples": [
- "my-sast-scanner"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the scanner, not required to be unique.",
- "minLength": 1,
- "examples": [
- "My SAST Scanner"
- ]
- },
- "url": {
- "type": "string",
- "description": "A link to more information about the scanner.",
- "examples": [
- "https://scanner.url"
- ]
- },
- "version": {
- "type": "string",
- "description": "The version of the scanner.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the scanner.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- }
- }
- },
- "start_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-02-14T16:01:59"
- ]
- },
- "status": {
- "type": "string",
- "description": "Result of the scan.",
- "enum": [
- "success",
- "failure"
- ]
- },
- "type": {
- "type": "string",
- "description": "Type of the scan.",
- "enum": [
- "sast"
- ]
- }
- }
- },
- "schema": {
- "type": "string",
- "description": "URI pointing to the validating security report schema.",
- "format": "uri"
- },
- "version": {
- "type": "string",
- "description": "The version of the schema to which the JSON report conforms.",
- "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
- },
- "vulnerabilities": {
- "type": "array",
- "description": "Array of vulnerability objects.",
- "items": {
- "type": "object",
- "description": "Describes the vulnerability.",
- "required": [
- "category",
- "cve",
- "identifiers",
- "location",
- "scanner"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
- "examples": [
- "642735a5-1425-428d-8d4e-3c854885a3c9"
- ]
- },
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
- "name": {
- "type": "string",
- "description": "The name of the vulnerability. This must not include the finding's specific information."
- },
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
- "description": {
- "type": "string",
- "description": "A long text section describing the vulnerability more fully."
- },
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
- "severity": {
- "type": "string",
- "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Info",
- "Unknown",
- "Low",
- "Medium",
- "High",
- "Critical"
- ]
- },
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
- "solution": {
- "type": "string",
- "description": "Explanation of how to fix the vulnerability."
- },
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
- "identifiers": {
- "type": "array",
- "minItems": 1,
- "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
- "items": {
- "type": "object",
- "required": [
- "type",
- "name",
- "value"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
- "minLength": 1
- },
- "name": {
- "type": "string",
- "description": "Human-readable name of the identifier.",
- "minLength": 1
- },
- "url": {
- "type": "string",
- "description": "URL of the identifier's documentation.",
- "format": "uri"
- },
- "value": {
- "type": "string",
- "description": "Value of the identifier, for matching purpose.",
- "minLength": 1
- }
- }
- }
- },
- "links": {
- "type": "array",
- "description": "An array of references to external documentation or articles that describe the vulnerability.",
- "items": {
- "type": "object",
- "required": [
- "url"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the vulnerability details link."
- },
- "url": {
- "type": "string",
- "description": "URL of the vulnerability details document.",
- "format": "uri"
- }
- }
- }
- },
- "details": {
- "$ref": "#/definitions/named_list/properties/items"
- },
- "tracking": {
- "description": "Describes how this vulnerability should be tracked as the project changes.",
- "oneOf": [
- {
- "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
- "required": [
- "items"
- ],
- "properties": {
- "type": {
- "const": "source"
- },
- "items": {
- "type": "array",
- "items": {
- "description": "An item that should be tracked using source-specific tracking methods.",
- "type": "object",
- "required": [
- "signatures"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the file that includes the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the file that includes the vulnerability."
- },
- "signatures": {
- "type": "array",
- "description": "An array of calculated tracking signatures for this tracking item.",
- "minItems": 1,
- "items": {
- "description": "A calculated tracking signature value and metadata.",
- "required": [
- "algorithm",
- "value"
- ],
- "properties": {
- "algorithm": {
- "type": "string",
- "description": "The algorithm used to generate the signature."
- },
- "value": {
- "type": "string",
- "description": "The result of this signature algorithm."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "Each tracking type must declare its own type."
- }
- }
- },
- "flags": {
- "description": "Flags that can be attached to vulnerabilities.",
- "type": "array",
- "items": {
- "type": "object",
- "description": "Informational flags identified and assigned to a vulnerability.",
- "required": [
- "type",
- "origin",
- "description"
- ],
- "properties": {
- "type": {
- "type": "string",
- "minLength": 1,
- "description": "Result of the scan.",
- "enum": [
- "flagged-as-likely-false-positive"
- ]
- },
- "origin": {
- "minLength": 1,
- "description": "Tool that issued the flag.",
- "type": "string"
- },
- "description": {
- "minLength": 1,
- "description": "What the flag is about.",
- "type": "string"
- }
- }
- }
- },
- "location": {
- "type": "object",
- "description": "Identifies the vulnerability's location.",
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the code affected by the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the code affected by the vulnerability."
- },
- "class": {
- "type": "string",
- "description": "Provides the name of the class where the vulnerability is located."
- },
- "method": {
- "type": "string",
- "description": "Provides the name of the method where the vulnerability is located."
- }
- }
- },
- "raw_source_code_extract": {
- "type": "string",
- "description": "Provides an unsanitized excerpt of the affected source code."
- }
- }
- }
- },
- "remediations": {
- "type": "array",
- "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
- "items": {
- "type": "object",
- "required": [
- "fixes",
- "summary",
- "diff"
- ],
- "properties": {
- "fixes": {
- "type": "array",
- "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
- "items": {
- "type": "object",
- "required": [
- "cve"
- ],
- "properties": {
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- }
- }
- }
- },
- "summary": {
- "type": "string",
- "minLength": 1,
- "description": "An overview of how the vulnerabilities were fixed."
- },
- "diff": {
- "type": "string",
- "minLength": 1,
- "description": "A base64-encoded remediation code diff, compatible with git apply."
- }
- }
- }
- }
- }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.3/secret-detection-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.3/secret-detection-report-format.json
deleted file mode 100644
index 5a315e39385..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.3/secret-detection-report-format.json
+++ /dev/null
@@ -1,892 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Report format for GitLab Secret Detection",
- "description": "This schema provides the the report format for the Secret Detection analyzer (https://docs.gitlab.com/ee/user/application_security/secret_detection)",
- "definitions": {
- "detail_type": {
- "oneOf": [
- {
- "$ref": "#/definitions/named_list"
- },
- {
- "$ref": "#/definitions/list"
- },
- {
- "$ref": "#/definitions/table"
- },
- {
- "$ref": "#/definitions/text"
- },
- {
- "$ref": "#/definitions/url"
- },
- {
- "$ref": "#/definitions/code"
- },
- {
- "$ref": "#/definitions/value"
- },
- {
- "$ref": "#/definitions/diff"
- },
- {
- "$ref": "#/definitions/markdown"
- },
- {
- "$ref": "#/definitions/commit"
- },
- {
- "$ref": "#/definitions/file_location"
- },
- {
- "$ref": "#/definitions/module_location"
- }
- ]
- },
- "text_value": {
- "type": "string"
- },
- "named_field": {
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "$ref": "#/definitions/text_value",
- "minLength": 1
- },
- "description": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "named_list": {
- "type": "object",
- "description": "An object with named and typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "named-list"
- },
- "items": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- {
- "$ref": "#/definitions/named_field"
- },
- {
- "$ref": "#/definitions/detail_type"
- }
- ]
- }
- }
- }
- }
- },
- "list": {
- "type": "object",
- "description": "A list of typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- },
- "table": {
- "type": "object",
- "description": "A table of typed fields",
- "required": [
- "type",
- "rows"
- ],
- "properties": {
- "type": {
- "const": "table"
- },
- "header": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- },
- "rows": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- }
- },
- "text": {
- "type": "object",
- "description": "Raw text",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "text"
- },
- "value": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "url": {
- "type": "object",
- "description": "A single URL",
- "required": [
- "type",
- "href"
- ],
- "properties": {
- "type": {
- "const": "url"
- },
- "text": {
- "$ref": "#/definitions/text_value"
- },
- "href": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "http://mysite.com"
- ]
- }
- }
- },
- "code": {
- "type": "object",
- "description": "A codeblock",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "code"
- },
- "value": {
- "type": "string"
- },
- "lang": {
- "type": "string",
- "description": "A programming language"
- }
- }
- },
- "value": {
- "type": "object",
- "description": "A field that can store a range of types of value",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "value"
- },
- "value": {
- "type": [
- "number",
- "string",
- "boolean"
- ]
- }
- }
- },
- "diff": {
- "type": "object",
- "description": "A diff",
- "required": [
- "type",
- "before",
- "after"
- ],
- "properties": {
- "type": {
- "const": "diff"
- },
- "before": {
- "type": "string"
- },
- "after": {
- "type": "string"
- }
- }
- },
- "markdown": {
- "type": "object",
- "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "markdown"
- },
- "value": {
- "$ref": "#/definitions/text_value",
- "examples": [
- "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
- ]
- }
- }
- },
- "commit": {
- "type": "object",
- "description": "A commit/tag/branch within the GitLab project",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "commit"
- },
- "value": {
- "type": "string",
- "description": "The commit SHA",
- "minLength": 1
- }
- }
- },
- "file_location": {
- "type": "object",
- "description": "A location within a file in the project",
- "required": [
- "type",
- "file_name",
- "line_start"
- ],
- "properties": {
- "type": {
- "const": "file-location"
- },
- "file_name": {
- "type": "string",
- "minLength": 1
- },
- "line_start": {
- "type": "integer"
- },
- "line_end": {
- "type": "integer"
- }
- }
- },
- "module_location": {
- "type": "object",
- "description": "A location within a binary module of the form module+relative_offset",
- "required": [
- "type",
- "module_name",
- "offset"
- ],
- "properties": {
- "type": {
- "const": "module-location"
- },
- "module_name": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "compiled_binary"
- ]
- },
- "offset": {
- "type": "integer",
- "examples": [
- 100
- ]
- }
- }
- }
- },
- "self": {
- "version": "14.0.3"
- },
- "required": [
- "version",
- "vulnerabilities"
- ],
- "additionalProperties": true,
- "properties": {
- "scan": {
- "type": "object",
- "required": [
- "end_time",
- "scanner",
- "start_time",
- "status",
- "type"
- ],
- "properties": {
- "end_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-01-28T03:26:02"
- ]
- },
- "messages": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Communication intended for the initiator of a scan.",
- "required": [
- "level",
- "value"
- ],
- "properties": {
- "level": {
- "type": "string",
- "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
- "enum": [
- "info",
- "warn",
- "fatal"
- ],
- "examples": [
- "info"
- ]
- },
- "value": {
- "type": "string",
- "description": "The message to communicate.",
- "minLength": 1,
- "examples": [
- "Permission denied, scanning aborted"
- ]
- }
- }
- }
- },
- "analyzer": {
- "type": "object",
- "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the analyzer.",
- "minLength": 1,
- "examples": [
- "gitlab-dast"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the analyzer, not required to be unique.",
- "minLength": 1,
- "examples": [
- "GitLab DAST"
- ]
- },
- "url": {
- "type": "string",
- "format": "uri",
- "pattern": "^https?://.+",
- "description": "A link to more information about the analyzer.",
- "examples": [
- "https://docs.gitlab.com/ee/user/application_security/dast"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the analyzer.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- },
- "version": {
- "type": "string",
- "description": "The version of the analyzer.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- }
- }
- },
- "scanner": {
- "type": "object",
- "description": "Object defining the scanner used to perform the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the scanner.",
- "minLength": 1,
- "examples": [
- "my-sast-scanner"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the scanner, not required to be unique.",
- "minLength": 1,
- "examples": [
- "My SAST Scanner"
- ]
- },
- "url": {
- "type": "string",
- "description": "A link to more information about the scanner.",
- "examples": [
- "https://scanner.url"
- ]
- },
- "version": {
- "type": "string",
- "description": "The version of the scanner.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the scanner.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- }
- }
- },
- "start_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-02-14T16:01:59"
- ]
- },
- "status": {
- "type": "string",
- "description": "Result of the scan.",
- "enum": [
- "success",
- "failure"
- ]
- },
- "type": {
- "type": "string",
- "description": "Type of the scan.",
- "enum": [
- "secret_detection"
- ]
- }
- }
- },
- "schema": {
- "type": "string",
- "description": "URI pointing to the validating security report schema.",
- "format": "uri"
- },
- "version": {
- "type": "string",
- "description": "The version of the schema to which the JSON report conforms.",
- "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
- },
- "vulnerabilities": {
- "type": "array",
- "description": "Array of vulnerability objects.",
- "items": {
- "type": "object",
- "description": "Describes the vulnerability.",
- "required": [
- "category",
- "cve",
- "identifiers",
- "location",
- "scanner"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
- "examples": [
- "642735a5-1425-428d-8d4e-3c854885a3c9"
- ]
- },
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
- "name": {
- "type": "string",
- "description": "The name of the vulnerability. This must not include the finding's specific information."
- },
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
- "description": {
- "type": "string",
- "description": "A long text section describing the vulnerability more fully."
- },
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
- "severity": {
- "type": "string",
- "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Info",
- "Unknown",
- "Low",
- "Medium",
- "High",
- "Critical"
- ]
- },
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
- "solution": {
- "type": "string",
- "description": "Explanation of how to fix the vulnerability."
- },
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
- "identifiers": {
- "type": "array",
- "minItems": 1,
- "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
- "items": {
- "type": "object",
- "required": [
- "type",
- "name",
- "value"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
- "minLength": 1
- },
- "name": {
- "type": "string",
- "description": "Human-readable name of the identifier.",
- "minLength": 1
- },
- "url": {
- "type": "string",
- "description": "URL of the identifier's documentation.",
- "format": "uri"
- },
- "value": {
- "type": "string",
- "description": "Value of the identifier, for matching purpose.",
- "minLength": 1
- }
- }
- }
- },
- "links": {
- "type": "array",
- "description": "An array of references to external documentation or articles that describe the vulnerability.",
- "items": {
- "type": "object",
- "required": [
- "url"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the vulnerability details link."
- },
- "url": {
- "type": "string",
- "description": "URL of the vulnerability details document.",
- "format": "uri"
- }
- }
- }
- },
- "details": {
- "$ref": "#/definitions/named_list/properties/items"
- },
- "tracking": {
- "description": "Describes how this vulnerability should be tracked as the project changes.",
- "oneOf": [
- {
- "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
- "required": [
- "items"
- ],
- "properties": {
- "type": {
- "const": "source"
- },
- "items": {
- "type": "array",
- "items": {
- "description": "An item that should be tracked using source-specific tracking methods.",
- "type": "object",
- "required": [
- "signatures"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the file that includes the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the file that includes the vulnerability."
- },
- "signatures": {
- "type": "array",
- "description": "An array of calculated tracking signatures for this tracking item.",
- "minItems": 1,
- "items": {
- "description": "A calculated tracking signature value and metadata.",
- "required": [
- "algorithm",
- "value"
- ],
- "properties": {
- "algorithm": {
- "type": "string",
- "description": "The algorithm used to generate the signature."
- },
- "value": {
- "type": "string",
- "description": "The result of this signature algorithm."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "Each tracking type must declare its own type."
- }
- }
- },
- "flags": {
- "description": "Flags that can be attached to vulnerabilities.",
- "type": "array",
- "items": {
- "type": "object",
- "description": "Informational flags identified and assigned to a vulnerability.",
- "required": [
- "type",
- "origin",
- "description"
- ],
- "properties": {
- "type": {
- "type": "string",
- "minLength": 1,
- "description": "Result of the scan.",
- "enum": [
- "flagged-as-likely-false-positive"
- ]
- },
- "origin": {
- "minLength": 1,
- "description": "Tool that issued the flag.",
- "type": "string"
- },
- "description": {
- "minLength": 1,
- "description": "What the flag is about.",
- "type": "string"
- }
- }
- }
- },
- "location": {
- "required": [
- "commit"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located"
- },
- "commit": {
- "type": "object",
- "description": "Represents the commit in which the vulnerability was detected",
- "required": [
- "sha"
- ],
- "properties": {
- "author": {
- "type": "string"
- },
- "date": {
- "type": "string"
- },
- "message": {
- "type": "string"
- },
- "sha": {
- "type": "string",
- "minLength": 1
- }
- }
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the code affected by the vulnerability"
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the code affected by the vulnerability"
- },
- "class": {
- "type": "string",
- "description": "Provides the name of the class where the vulnerability is located"
- },
- "method": {
- "type": "string",
- "description": "Provides the name of the method where the vulnerability is located"
- }
- }
- },
- "raw_source_code_extract": {
- "type": "string",
- "description": "Provides an unsanitized excerpt of the affected source code."
- }
- }
- }
- },
- "remediations": {
- "type": "array",
- "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
- "items": {
- "type": "object",
- "required": [
- "fixes",
- "summary",
- "diff"
- ],
- "properties": {
- "fixes": {
- "type": "array",
- "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
- "items": {
- "type": "object",
- "required": [
- "cve"
- ],
- "properties": {
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- }
- }
- }
- },
- "summary": {
- "type": "string",
- "minLength": 1,
- "description": "An overview of how the vulnerabilities were fixed."
- },
- "diff": {
- "type": "string",
- "minLength": 1,
- "description": "A base64-encoded remediation code diff, compatible with git apply."
- }
- }
- }
- }
- }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.4/cluster-image-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.4/cluster-image-scanning-report-format.json
deleted file mode 100644
index 3736eac0ba0..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.4/cluster-image-scanning-report-format.json
+++ /dev/null
@@ -1,977 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Report format for GitLab Cluster Image Scanning",
- "description": "This schema provides the the report format for Cluster Image Scanning (https://docs.gitlab.com/ee/user/application_security/cluster_image_scanning/).",
- "definitions": {
- "detail_type": {
- "oneOf": [
- {
- "$ref": "#/definitions/named_list"
- },
- {
- "$ref": "#/definitions/list"
- },
- {
- "$ref": "#/definitions/table"
- },
- {
- "$ref": "#/definitions/text"
- },
- {
- "$ref": "#/definitions/url"
- },
- {
- "$ref": "#/definitions/code"
- },
- {
- "$ref": "#/definitions/value"
- },
- {
- "$ref": "#/definitions/diff"
- },
- {
- "$ref": "#/definitions/markdown"
- },
- {
- "$ref": "#/definitions/commit"
- },
- {
- "$ref": "#/definitions/file_location"
- },
- {
- "$ref": "#/definitions/module_location"
- }
- ]
- },
- "text_value": {
- "type": "string"
- },
- "named_field": {
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "$ref": "#/definitions/text_value",
- "minLength": 1
- },
- "description": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "named_list": {
- "type": "object",
- "description": "An object with named and typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "named-list"
- },
- "items": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- {
- "$ref": "#/definitions/named_field"
- },
- {
- "$ref": "#/definitions/detail_type"
- }
- ]
- }
- }
- }
- }
- },
- "list": {
- "type": "object",
- "description": "A list of typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- },
- "table": {
- "type": "object",
- "description": "A table of typed fields",
- "required": [
- "type",
- "rows"
- ],
- "properties": {
- "type": {
- "const": "table"
- },
- "header": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- },
- "rows": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- }
- },
- "text": {
- "type": "object",
- "description": "Raw text",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "text"
- },
- "value": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "url": {
- "type": "object",
- "description": "A single URL",
- "required": [
- "type",
- "href"
- ],
- "properties": {
- "type": {
- "const": "url"
- },
- "text": {
- "$ref": "#/definitions/text_value"
- },
- "href": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "http://mysite.com"
- ]
- }
- }
- },
- "code": {
- "type": "object",
- "description": "A codeblock",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "code"
- },
- "value": {
- "type": "string"
- },
- "lang": {
- "type": "string",
- "description": "A programming language"
- }
- }
- },
- "value": {
- "type": "object",
- "description": "A field that can store a range of types of value",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "value"
- },
- "value": {
- "type": [
- "number",
- "string",
- "boolean"
- ]
- }
- }
- },
- "diff": {
- "type": "object",
- "description": "A diff",
- "required": [
- "type",
- "before",
- "after"
- ],
- "properties": {
- "type": {
- "const": "diff"
- },
- "before": {
- "type": "string"
- },
- "after": {
- "type": "string"
- }
- }
- },
- "markdown": {
- "type": "object",
- "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "markdown"
- },
- "value": {
- "$ref": "#/definitions/text_value",
- "examples": [
- "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
- ]
- }
- }
- },
- "commit": {
- "type": "object",
- "description": "A commit/tag/branch within the GitLab project",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "commit"
- },
- "value": {
- "type": "string",
- "description": "The commit SHA",
- "minLength": 1
- }
- }
- },
- "file_location": {
- "type": "object",
- "description": "A location within a file in the project",
- "required": [
- "type",
- "file_name",
- "line_start"
- ],
- "properties": {
- "type": {
- "const": "file-location"
- },
- "file_name": {
- "type": "string",
- "minLength": 1
- },
- "line_start": {
- "type": "integer"
- },
- "line_end": {
- "type": "integer"
- }
- }
- },
- "module_location": {
- "type": "object",
- "description": "A location within a binary module of the form module+relative_offset",
- "required": [
- "type",
- "module_name",
- "offset"
- ],
- "properties": {
- "type": {
- "const": "module-location"
- },
- "module_name": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "compiled_binary"
- ]
- },
- "offset": {
- "type": "integer",
- "examples": [
- 100
- ]
- }
- }
- }
- },
- "self": {
- "version": "14.0.4"
- },
- "required": [
- "version",
- "vulnerabilities"
- ],
- "additionalProperties": true,
- "properties": {
- "scan": {
- "type": "object",
- "required": [
- "end_time",
- "scanner",
- "start_time",
- "status",
- "type"
- ],
- "properties": {
- "end_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-01-28T03:26:02"
- ]
- },
- "messages": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Communication intended for the initiator of a scan.",
- "required": [
- "level",
- "value"
- ],
- "properties": {
- "level": {
- "type": "string",
- "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
- "enum": [
- "info",
- "warn",
- "fatal"
- ],
- "examples": [
- "info"
- ]
- },
- "value": {
- "type": "string",
- "description": "The message to communicate.",
- "minLength": 1,
- "examples": [
- "Permission denied, scanning aborted"
- ]
- }
- }
- }
- },
- "analyzer": {
- "type": "object",
- "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the analyzer.",
- "minLength": 1,
- "examples": [
- "gitlab-dast"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the analyzer, not required to be unique.",
- "minLength": 1,
- "examples": [
- "GitLab DAST"
- ]
- },
- "url": {
- "type": "string",
- "format": "uri",
- "pattern": "^https?://.+",
- "description": "A link to more information about the analyzer.",
- "examples": [
- "https://docs.gitlab.com/ee/user/application_security/dast"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the analyzer.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- },
- "version": {
- "type": "string",
- "description": "The version of the analyzer.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- }
- }
- },
- "scanner": {
- "type": "object",
- "description": "Object defining the scanner used to perform the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the scanner.",
- "minLength": 1,
- "examples": [
- "my-sast-scanner"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the scanner, not required to be unique.",
- "minLength": 1,
- "examples": [
- "My SAST Scanner"
- ]
- },
- "url": {
- "type": "string",
- "description": "A link to more information about the scanner.",
- "examples": [
- "https://scanner.url"
- ]
- },
- "version": {
- "type": "string",
- "description": "The version of the scanner.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the scanner.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- }
- }
- },
- "start_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-02-14T16:01:59"
- ]
- },
- "status": {
- "type": "string",
- "description": "Result of the scan.",
- "enum": [
- "success",
- "failure"
- ]
- },
- "type": {
- "type": "string",
- "description": "Type of the scan.",
- "enum": [
- "cluster_image_scanning"
- ]
- }
- }
- },
- "schema": {
- "type": "string",
- "description": "URI pointing to the validating security report schema.",
- "format": "uri"
- },
- "version": {
- "type": "string",
- "description": "The version of the schema to which the JSON report conforms.",
- "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
- },
- "vulnerabilities": {
- "type": "array",
- "description": "Array of vulnerability objects.",
- "items": {
- "type": "object",
- "description": "Describes the vulnerability.",
- "required": [
- "category",
- "cve",
- "identifiers",
- "location",
- "scanner"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
- "examples": [
- "642735a5-1425-428d-8d4e-3c854885a3c9"
- ]
- },
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
- "name": {
- "type": "string",
- "description": "The name of the vulnerability. This must not include the finding's specific information."
- },
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
- "description": {
- "type": "string",
- "description": "A long text section describing the vulnerability more fully."
- },
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
- "severity": {
- "type": "string",
- "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Info",
- "Unknown",
- "Low",
- "Medium",
- "High",
- "Critical"
- ]
- },
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
- "solution": {
- "type": "string",
- "description": "Explanation of how to fix the vulnerability."
- },
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
- "identifiers": {
- "type": "array",
- "minItems": 1,
- "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
- "items": {
- "type": "object",
- "required": [
- "type",
- "name",
- "value"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
- "minLength": 1
- },
- "name": {
- "type": "string",
- "description": "Human-readable name of the identifier.",
- "minLength": 1
- },
- "url": {
- "type": "string",
- "description": "URL of the identifier's documentation.",
- "format": "uri"
- },
- "value": {
- "type": "string",
- "description": "Value of the identifier, for matching purpose.",
- "minLength": 1
- }
- }
- }
- },
- "links": {
- "type": "array",
- "description": "An array of references to external documentation or articles that describe the vulnerability.",
- "items": {
- "type": "object",
- "required": [
- "url"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the vulnerability details link."
- },
- "url": {
- "type": "string",
- "description": "URL of the vulnerability details document.",
- "format": "uri"
- }
- }
- }
- },
- "details": {
- "$ref": "#/definitions/named_list/properties/items"
- },
- "tracking": {
- "description": "Describes how this vulnerability should be tracked as the project changes.",
- "oneOf": [
- {
- "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
- "required": [
- "items"
- ],
- "properties": {
- "type": {
- "const": "source"
- },
- "items": {
- "type": "array",
- "items": {
- "description": "An item that should be tracked using source-specific tracking methods.",
- "type": "object",
- "required": [
- "signatures"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the file that includes the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the file that includes the vulnerability."
- },
- "signatures": {
- "type": "array",
- "description": "An array of calculated tracking signatures for this tracking item.",
- "minItems": 1,
- "items": {
- "description": "A calculated tracking signature value and metadata.",
- "required": [
- "algorithm",
- "value"
- ],
- "properties": {
- "algorithm": {
- "type": "string",
- "description": "The algorithm used to generate the signature."
- },
- "value": {
- "type": "string",
- "description": "The result of this signature algorithm."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "Each tracking type must declare its own type."
- }
- }
- },
- "flags": {
- "description": "Flags that can be attached to vulnerabilities.",
- "type": "array",
- "items": {
- "type": "object",
- "description": "Informational flags identified and assigned to a vulnerability.",
- "required": [
- "type",
- "origin",
- "description"
- ],
- "properties": {
- "type": {
- "type": "string",
- "minLength": 1,
- "description": "Result of the scan.",
- "enum": [
- "flagged-as-likely-false-positive"
- ]
- },
- "origin": {
- "minLength": 1,
- "description": "Tool that issued the flag.",
- "type": "string"
- },
- "description": {
- "minLength": 1,
- "description": "What the flag is about.",
- "type": "string"
- }
- }
- }
- },
- "location": {
- "type": "object",
- "description": "Identifies the vulnerability's location.",
- "required": [
- "dependency",
- "image",
- "kubernetes_resource"
- ],
- "properties": {
- "dependency": {
- "type": "object",
- "description": "Describes the dependency of a project where the vulnerability is located.",
- "properties": {
- "package": {
- "type": "object",
- "description": "Provides information on the package where the vulnerability is located.",
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the package where the vulnerability is located."
- }
- }
- },
- "version": {
- "type": "string",
- "description": "Version of the vulnerable package."
- },
- "iid": {
- "description": "ID that identifies the dependency in the scope of a dependency file.",
- "type": "number"
- },
- "direct": {
- "type": "boolean",
- "description": "Tells whether this is a direct, top-level dependency of the scanned project."
- },
- "dependency_path": {
- "type": "array",
- "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.",
- "items": {
- "type": "object",
- "required": [
- "iid"
- ],
- "properties": {
- "iid": {
- "type": "number",
- "description": "ID that is unique in the scope of a parent object, and specific to the resource type."
- }
- }
- }
- }
- }
- },
- "operating_system": {
- "type": "string",
- "minLength": 1,
- "maxLength": 255,
- "description": "The operating system that contains the vulnerable package."
- },
- "image": {
- "type": "string",
- "minLength": 1,
- "description": "The analyzed Docker image.",
- "examples": [
- "index.docker.io/library/nginx:1.21"
- ]
- },
- "kubernetes_resource": {
- "type": "object",
- "description": "The specific Kubernetes resource that was scanned.",
- "required": [
- "namespace",
- "kind",
- "name",
- "container_name"
- ],
- "properties": {
- "namespace": {
- "type": "string",
- "minLength": 1,
- "maxLength": 255,
- "description": "The Kubernetes namespace the resource that had its image scanned.",
- "examples": [
- "default",
- "staging",
- "production"
- ]
- },
- "kind": {
- "type": "string",
- "minLength": 1,
- "maxLength": 255,
- "description": "The Kubernetes kind the resource that had its image scanned.",
- "examples": [
- "Deployment",
- "DaemonSet"
- ]
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "maxLength": 255,
- "description": "The name of the resource that had its image scanned.",
- "examples": [
- "nginx-ingress"
- ]
- },
- "container_name": {
- "type": "string",
- "minLength": 1,
- "maxLength": 255,
- "description": "The name of the container that had its image scanned.",
- "examples": [
- "nginx"
- ]
- },
- "agent_id": {
- "type": "string",
- "minLength": 1,
- "maxLength": 255,
- "description": "The GitLab ID of the Kubernetes Agent which performed the scan.",
- "examples": [
- "1234"
- ]
- },
- "cluster_id": {
- "type": "string",
- "minLength": 1,
- "maxLength": 255,
- "description": "The GitLab ID of the Kubernetes cluster when using cluster integration.",
- "examples": [
- "1234"
- ]
- }
- }
- }
- }
- }
- }
- }
- },
- "remediations": {
- "type": "array",
- "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
- "items": {
- "type": "object",
- "required": [
- "fixes",
- "summary",
- "diff"
- ],
- "properties": {
- "fixes": {
- "type": "array",
- "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
- "items": {
- "type": "object",
- "required": [
- "cve"
- ],
- "properties": {
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- }
- }
- }
- },
- "summary": {
- "type": "string",
- "minLength": 1,
- "description": "An overview of how the vulnerabilities were fixed."
- },
- "diff": {
- "type": "string",
- "minLength": 1,
- "description": "A base64-encoded remediation code diff, compatible with git apply."
- }
- }
- }
- }
- }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.4/container-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.4/container-scanning-report-format.json
deleted file mode 100644
index e324201b04b..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.4/container-scanning-report-format.json
+++ /dev/null
@@ -1,904 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Report format for GitLab Container Scanning",
- "description": "This schema provides the the report format for Container Scanning (https://docs.gitlab.com/ee/user/application_security/container_scanning).",
- "definitions": {
- "detail_type": {
- "oneOf": [
- {
- "$ref": "#/definitions/named_list"
- },
- {
- "$ref": "#/definitions/list"
- },
- {
- "$ref": "#/definitions/table"
- },
- {
- "$ref": "#/definitions/text"
- },
- {
- "$ref": "#/definitions/url"
- },
- {
- "$ref": "#/definitions/code"
- },
- {
- "$ref": "#/definitions/value"
- },
- {
- "$ref": "#/definitions/diff"
- },
- {
- "$ref": "#/definitions/markdown"
- },
- {
- "$ref": "#/definitions/commit"
- },
- {
- "$ref": "#/definitions/file_location"
- },
- {
- "$ref": "#/definitions/module_location"
- }
- ]
- },
- "text_value": {
- "type": "string"
- },
- "named_field": {
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "$ref": "#/definitions/text_value",
- "minLength": 1
- },
- "description": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "named_list": {
- "type": "object",
- "description": "An object with named and typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "named-list"
- },
- "items": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- {
- "$ref": "#/definitions/named_field"
- },
- {
- "$ref": "#/definitions/detail_type"
- }
- ]
- }
- }
- }
- }
- },
- "list": {
- "type": "object",
- "description": "A list of typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- },
- "table": {
- "type": "object",
- "description": "A table of typed fields",
- "required": [
- "type",
- "rows"
- ],
- "properties": {
- "type": {
- "const": "table"
- },
- "header": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- },
- "rows": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- }
- },
- "text": {
- "type": "object",
- "description": "Raw text",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "text"
- },
- "value": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "url": {
- "type": "object",
- "description": "A single URL",
- "required": [
- "type",
- "href"
- ],
- "properties": {
- "type": {
- "const": "url"
- },
- "text": {
- "$ref": "#/definitions/text_value"
- },
- "href": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "http://mysite.com"
- ]
- }
- }
- },
- "code": {
- "type": "object",
- "description": "A codeblock",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "code"
- },
- "value": {
- "type": "string"
- },
- "lang": {
- "type": "string",
- "description": "A programming language"
- }
- }
- },
- "value": {
- "type": "object",
- "description": "A field that can store a range of types of value",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "value"
- },
- "value": {
- "type": [
- "number",
- "string",
- "boolean"
- ]
- }
- }
- },
- "diff": {
- "type": "object",
- "description": "A diff",
- "required": [
- "type",
- "before",
- "after"
- ],
- "properties": {
- "type": {
- "const": "diff"
- },
- "before": {
- "type": "string"
- },
- "after": {
- "type": "string"
- }
- }
- },
- "markdown": {
- "type": "object",
- "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "markdown"
- },
- "value": {
- "$ref": "#/definitions/text_value",
- "examples": [
- "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
- ]
- }
- }
- },
- "commit": {
- "type": "object",
- "description": "A commit/tag/branch within the GitLab project",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "commit"
- },
- "value": {
- "type": "string",
- "description": "The commit SHA",
- "minLength": 1
- }
- }
- },
- "file_location": {
- "type": "object",
- "description": "A location within a file in the project",
- "required": [
- "type",
- "file_name",
- "line_start"
- ],
- "properties": {
- "type": {
- "const": "file-location"
- },
- "file_name": {
- "type": "string",
- "minLength": 1
- },
- "line_start": {
- "type": "integer"
- },
- "line_end": {
- "type": "integer"
- }
- }
- },
- "module_location": {
- "type": "object",
- "description": "A location within a binary module of the form module+relative_offset",
- "required": [
- "type",
- "module_name",
- "offset"
- ],
- "properties": {
- "type": {
- "const": "module-location"
- },
- "module_name": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "compiled_binary"
- ]
- },
- "offset": {
- "type": "integer",
- "examples": [
- 100
- ]
- }
- }
- }
- },
- "self": {
- "version": "14.0.4"
- },
- "required": [
- "version",
- "vulnerabilities"
- ],
- "additionalProperties": true,
- "properties": {
- "scan": {
- "type": "object",
- "required": [
- "end_time",
- "scanner",
- "start_time",
- "status",
- "type"
- ],
- "properties": {
- "end_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-01-28T03:26:02"
- ]
- },
- "messages": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Communication intended for the initiator of a scan.",
- "required": [
- "level",
- "value"
- ],
- "properties": {
- "level": {
- "type": "string",
- "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
- "enum": [
- "info",
- "warn",
- "fatal"
- ],
- "examples": [
- "info"
- ]
- },
- "value": {
- "type": "string",
- "description": "The message to communicate.",
- "minLength": 1,
- "examples": [
- "Permission denied, scanning aborted"
- ]
- }
- }
- }
- },
- "analyzer": {
- "type": "object",
- "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the analyzer.",
- "minLength": 1,
- "examples": [
- "gitlab-dast"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the analyzer, not required to be unique.",
- "minLength": 1,
- "examples": [
- "GitLab DAST"
- ]
- },
- "url": {
- "type": "string",
- "format": "uri",
- "pattern": "^https?://.+",
- "description": "A link to more information about the analyzer.",
- "examples": [
- "https://docs.gitlab.com/ee/user/application_security/dast"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the analyzer.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- },
- "version": {
- "type": "string",
- "description": "The version of the analyzer.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- }
- }
- },
- "scanner": {
- "type": "object",
- "description": "Object defining the scanner used to perform the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the scanner.",
- "minLength": 1,
- "examples": [
- "my-sast-scanner"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the scanner, not required to be unique.",
- "minLength": 1,
- "examples": [
- "My SAST Scanner"
- ]
- },
- "url": {
- "type": "string",
- "description": "A link to more information about the scanner.",
- "examples": [
- "https://scanner.url"
- ]
- },
- "version": {
- "type": "string",
- "description": "The version of the scanner.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the scanner.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- }
- }
- },
- "start_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-02-14T16:01:59"
- ]
- },
- "status": {
- "type": "string",
- "description": "Result of the scan.",
- "enum": [
- "success",
- "failure"
- ]
- },
- "type": {
- "type": "string",
- "description": "Type of the scan.",
- "enum": [
- "container_scanning"
- ]
- }
- }
- },
- "schema": {
- "type": "string",
- "description": "URI pointing to the validating security report schema.",
- "format": "uri"
- },
- "version": {
- "type": "string",
- "description": "The version of the schema to which the JSON report conforms.",
- "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
- },
- "vulnerabilities": {
- "type": "array",
- "description": "Array of vulnerability objects.",
- "items": {
- "type": "object",
- "description": "Describes the vulnerability.",
- "required": [
- "category",
- "cve",
- "identifiers",
- "location",
- "scanner"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
- "examples": [
- "642735a5-1425-428d-8d4e-3c854885a3c9"
- ]
- },
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
- "name": {
- "type": "string",
- "description": "The name of the vulnerability. This must not include the finding's specific information."
- },
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
- "description": {
- "type": "string",
- "description": "A long text section describing the vulnerability more fully."
- },
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
- "severity": {
- "type": "string",
- "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Info",
- "Unknown",
- "Low",
- "Medium",
- "High",
- "Critical"
- ]
- },
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
- "solution": {
- "type": "string",
- "description": "Explanation of how to fix the vulnerability."
- },
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
- "identifiers": {
- "type": "array",
- "minItems": 1,
- "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
- "items": {
- "type": "object",
- "required": [
- "type",
- "name",
- "value"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
- "minLength": 1
- },
- "name": {
- "type": "string",
- "description": "Human-readable name of the identifier.",
- "minLength": 1
- },
- "url": {
- "type": "string",
- "description": "URL of the identifier's documentation.",
- "format": "uri"
- },
- "value": {
- "type": "string",
- "description": "Value of the identifier, for matching purpose.",
- "minLength": 1
- }
- }
- }
- },
- "links": {
- "type": "array",
- "description": "An array of references to external documentation or articles that describe the vulnerability.",
- "items": {
- "type": "object",
- "required": [
- "url"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the vulnerability details link."
- },
- "url": {
- "type": "string",
- "description": "URL of the vulnerability details document.",
- "format": "uri"
- }
- }
- }
- },
- "details": {
- "$ref": "#/definitions/named_list/properties/items"
- },
- "tracking": {
- "description": "Describes how this vulnerability should be tracked as the project changes.",
- "oneOf": [
- {
- "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
- "required": [
- "items"
- ],
- "properties": {
- "type": {
- "const": "source"
- },
- "items": {
- "type": "array",
- "items": {
- "description": "An item that should be tracked using source-specific tracking methods.",
- "type": "object",
- "required": [
- "signatures"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the file that includes the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the file that includes the vulnerability."
- },
- "signatures": {
- "type": "array",
- "description": "An array of calculated tracking signatures for this tracking item.",
- "minItems": 1,
- "items": {
- "description": "A calculated tracking signature value and metadata.",
- "required": [
- "algorithm",
- "value"
- ],
- "properties": {
- "algorithm": {
- "type": "string",
- "description": "The algorithm used to generate the signature."
- },
- "value": {
- "type": "string",
- "description": "The result of this signature algorithm."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "Each tracking type must declare its own type."
- }
- }
- },
- "flags": {
- "description": "Flags that can be attached to vulnerabilities.",
- "type": "array",
- "items": {
- "type": "object",
- "description": "Informational flags identified and assigned to a vulnerability.",
- "required": [
- "type",
- "origin",
- "description"
- ],
- "properties": {
- "type": {
- "type": "string",
- "minLength": 1,
- "description": "Result of the scan.",
- "enum": [
- "flagged-as-likely-false-positive"
- ]
- },
- "origin": {
- "minLength": 1,
- "description": "Tool that issued the flag.",
- "type": "string"
- },
- "description": {
- "minLength": 1,
- "description": "What the flag is about.",
- "type": "string"
- }
- }
- }
- },
- "location": {
- "type": "object",
- "description": "Identifies the vulnerability's location.",
- "required": [
- "dependency",
- "operating_system",
- "image"
- ],
- "properties": {
- "dependency": {
- "type": "object",
- "description": "Describes the dependency of a project where the vulnerability is located.",
- "properties": {
- "package": {
- "type": "object",
- "description": "Provides information on the package where the vulnerability is located.",
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the package where the vulnerability is located."
- }
- }
- },
- "version": {
- "type": "string",
- "description": "Version of the vulnerable package."
- },
- "iid": {
- "description": "ID that identifies the dependency in the scope of a dependency file.",
- "type": "number"
- },
- "direct": {
- "type": "boolean",
- "description": "Tells whether this is a direct, top-level dependency of the scanned project."
- },
- "dependency_path": {
- "type": "array",
- "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.",
- "items": {
- "type": "object",
- "required": [
- "iid"
- ],
- "properties": {
- "iid": {
- "type": "number",
- "description": "ID that is unique in the scope of a parent object, and specific to the resource type."
- }
- }
- }
- }
- }
- },
- "operating_system": {
- "type": "string",
- "minLength": 1,
- "description": "The operating system that contains the vulnerable package."
- },
- "image": {
- "type": "string",
- "minLength": 1,
- "description": "The analyzed Docker image."
- }
- }
- }
- }
- }
- },
- "remediations": {
- "type": "array",
- "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
- "items": {
- "type": "object",
- "required": [
- "fixes",
- "summary",
- "diff"
- ],
- "properties": {
- "fixes": {
- "type": "array",
- "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
- "items": {
- "type": "object",
- "required": [
- "cve"
- ],
- "properties": {
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- }
- }
- }
- },
- "summary": {
- "type": "string",
- "minLength": 1,
- "description": "An overview of how the vulnerabilities were fixed."
- },
- "diff": {
- "type": "string",
- "minLength": 1,
- "description": "A base64-encoded remediation code diff, compatible with git apply."
- }
- }
- }
- }
- }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.4/coverage-fuzzing-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.4/coverage-fuzzing-report-format.json
deleted file mode 100644
index 7ac5d2b7783..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.4/coverage-fuzzing-report-format.json
+++ /dev/null
@@ -1,874 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Report format for GitLab Fuzz Testing",
- "description": "This schema provides the report format for Coverage Guided Fuzz Testing (https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing).",
- "definitions": {
- "detail_type": {
- "oneOf": [
- {
- "$ref": "#/definitions/named_list"
- },
- {
- "$ref": "#/definitions/list"
- },
- {
- "$ref": "#/definitions/table"
- },
- {
- "$ref": "#/definitions/text"
- },
- {
- "$ref": "#/definitions/url"
- },
- {
- "$ref": "#/definitions/code"
- },
- {
- "$ref": "#/definitions/value"
- },
- {
- "$ref": "#/definitions/diff"
- },
- {
- "$ref": "#/definitions/markdown"
- },
- {
- "$ref": "#/definitions/commit"
- },
- {
- "$ref": "#/definitions/file_location"
- },
- {
- "$ref": "#/definitions/module_location"
- }
- ]
- },
- "text_value": {
- "type": "string"
- },
- "named_field": {
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "$ref": "#/definitions/text_value",
- "minLength": 1
- },
- "description": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "named_list": {
- "type": "object",
- "description": "An object with named and typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "named-list"
- },
- "items": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- {
- "$ref": "#/definitions/named_field"
- },
- {
- "$ref": "#/definitions/detail_type"
- }
- ]
- }
- }
- }
- }
- },
- "list": {
- "type": "object",
- "description": "A list of typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- },
- "table": {
- "type": "object",
- "description": "A table of typed fields",
- "required": [
- "type",
- "rows"
- ],
- "properties": {
- "type": {
- "const": "table"
- },
- "header": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- },
- "rows": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- }
- },
- "text": {
- "type": "object",
- "description": "Raw text",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "text"
- },
- "value": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "url": {
- "type": "object",
- "description": "A single URL",
- "required": [
- "type",
- "href"
- ],
- "properties": {
- "type": {
- "const": "url"
- },
- "text": {
- "$ref": "#/definitions/text_value"
- },
- "href": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "http://mysite.com"
- ]
- }
- }
- },
- "code": {
- "type": "object",
- "description": "A codeblock",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "code"
- },
- "value": {
- "type": "string"
- },
- "lang": {
- "type": "string",
- "description": "A programming language"
- }
- }
- },
- "value": {
- "type": "object",
- "description": "A field that can store a range of types of value",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "value"
- },
- "value": {
- "type": [
- "number",
- "string",
- "boolean"
- ]
- }
- }
- },
- "diff": {
- "type": "object",
- "description": "A diff",
- "required": [
- "type",
- "before",
- "after"
- ],
- "properties": {
- "type": {
- "const": "diff"
- },
- "before": {
- "type": "string"
- },
- "after": {
- "type": "string"
- }
- }
- },
- "markdown": {
- "type": "object",
- "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "markdown"
- },
- "value": {
- "$ref": "#/definitions/text_value",
- "examples": [
- "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
- ]
- }
- }
- },
- "commit": {
- "type": "object",
- "description": "A commit/tag/branch within the GitLab project",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "commit"
- },
- "value": {
- "type": "string",
- "description": "The commit SHA",
- "minLength": 1
- }
- }
- },
- "file_location": {
- "type": "object",
- "description": "A location within a file in the project",
- "required": [
- "type",
- "file_name",
- "line_start"
- ],
- "properties": {
- "type": {
- "const": "file-location"
- },
- "file_name": {
- "type": "string",
- "minLength": 1
- },
- "line_start": {
- "type": "integer"
- },
- "line_end": {
- "type": "integer"
- }
- }
- },
- "module_location": {
- "type": "object",
- "description": "A location within a binary module of the form module+relative_offset",
- "required": [
- "type",
- "module_name",
- "offset"
- ],
- "properties": {
- "type": {
- "const": "module-location"
- },
- "module_name": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "compiled_binary"
- ]
- },
- "offset": {
- "type": "integer",
- "examples": [
- 100
- ]
- }
- }
- }
- },
- "self": {
- "version": "14.0.4"
- },
- "required": [
- "version",
- "vulnerabilities"
- ],
- "additionalProperties": true,
- "properties": {
- "scan": {
- "type": "object",
- "required": [
- "end_time",
- "scanner",
- "start_time",
- "status",
- "type"
- ],
- "properties": {
- "end_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-01-28T03:26:02"
- ]
- },
- "messages": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Communication intended for the initiator of a scan.",
- "required": [
- "level",
- "value"
- ],
- "properties": {
- "level": {
- "type": "string",
- "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
- "enum": [
- "info",
- "warn",
- "fatal"
- ],
- "examples": [
- "info"
- ]
- },
- "value": {
- "type": "string",
- "description": "The message to communicate.",
- "minLength": 1,
- "examples": [
- "Permission denied, scanning aborted"
- ]
- }
- }
- }
- },
- "analyzer": {
- "type": "object",
- "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the analyzer.",
- "minLength": 1,
- "examples": [
- "gitlab-dast"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the analyzer, not required to be unique.",
- "minLength": 1,
- "examples": [
- "GitLab DAST"
- ]
- },
- "url": {
- "type": "string",
- "format": "uri",
- "pattern": "^https?://.+",
- "description": "A link to more information about the analyzer.",
- "examples": [
- "https://docs.gitlab.com/ee/user/application_security/dast"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the analyzer.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- },
- "version": {
- "type": "string",
- "description": "The version of the analyzer.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- }
- }
- },
- "scanner": {
- "type": "object",
- "description": "Object defining the scanner used to perform the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the scanner.",
- "minLength": 1,
- "examples": [
- "my-sast-scanner"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the scanner, not required to be unique.",
- "minLength": 1,
- "examples": [
- "My SAST Scanner"
- ]
- },
- "url": {
- "type": "string",
- "description": "A link to more information about the scanner.",
- "examples": [
- "https://scanner.url"
- ]
- },
- "version": {
- "type": "string",
- "description": "The version of the scanner.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the scanner.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- }
- }
- },
- "start_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-02-14T16:01:59"
- ]
- },
- "status": {
- "type": "string",
- "description": "Result of the scan.",
- "enum": [
- "success",
- "failure"
- ]
- },
- "type": {
- "type": "string",
- "description": "Type of the scan.",
- "enum": [
- "coverage_fuzzing"
- ]
- }
- }
- },
- "schema": {
- "type": "string",
- "description": "URI pointing to the validating security report schema.",
- "format": "uri"
- },
- "version": {
- "type": "string",
- "description": "The version of the schema to which the JSON report conforms.",
- "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
- },
- "vulnerabilities": {
- "type": "array",
- "description": "Array of vulnerability objects.",
- "items": {
- "type": "object",
- "description": "Describes the vulnerability.",
- "required": [
- "category",
- "cve",
- "identifiers",
- "location",
- "scanner"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
- "examples": [
- "642735a5-1425-428d-8d4e-3c854885a3c9"
- ]
- },
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
- "name": {
- "type": "string",
- "description": "The name of the vulnerability. This must not include the finding's specific information."
- },
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
- "description": {
- "type": "string",
- "description": "A long text section describing the vulnerability more fully."
- },
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
- "severity": {
- "type": "string",
- "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Info",
- "Unknown",
- "Low",
- "Medium",
- "High",
- "Critical"
- ]
- },
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
- "solution": {
- "type": "string",
- "description": "Explanation of how to fix the vulnerability."
- },
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
- "identifiers": {
- "type": "array",
- "minItems": 1,
- "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
- "items": {
- "type": "object",
- "required": [
- "type",
- "name",
- "value"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
- "minLength": 1
- },
- "name": {
- "type": "string",
- "description": "Human-readable name of the identifier.",
- "minLength": 1
- },
- "url": {
- "type": "string",
- "description": "URL of the identifier's documentation.",
- "format": "uri"
- },
- "value": {
- "type": "string",
- "description": "Value of the identifier, for matching purpose.",
- "minLength": 1
- }
- }
- }
- },
- "links": {
- "type": "array",
- "description": "An array of references to external documentation or articles that describe the vulnerability.",
- "items": {
- "type": "object",
- "required": [
- "url"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the vulnerability details link."
- },
- "url": {
- "type": "string",
- "description": "URL of the vulnerability details document.",
- "format": "uri"
- }
- }
- }
- },
- "details": {
- "$ref": "#/definitions/named_list/properties/items"
- },
- "tracking": {
- "description": "Describes how this vulnerability should be tracked as the project changes.",
- "oneOf": [
- {
- "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
- "required": [
- "items"
- ],
- "properties": {
- "type": {
- "const": "source"
- },
- "items": {
- "type": "array",
- "items": {
- "description": "An item that should be tracked using source-specific tracking methods.",
- "type": "object",
- "required": [
- "signatures"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the file that includes the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the file that includes the vulnerability."
- },
- "signatures": {
- "type": "array",
- "description": "An array of calculated tracking signatures for this tracking item.",
- "minItems": 1,
- "items": {
- "description": "A calculated tracking signature value and metadata.",
- "required": [
- "algorithm",
- "value"
- ],
- "properties": {
- "algorithm": {
- "type": "string",
- "description": "The algorithm used to generate the signature."
- },
- "value": {
- "type": "string",
- "description": "The result of this signature algorithm."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "Each tracking type must declare its own type."
- }
- }
- },
- "flags": {
- "description": "Flags that can be attached to vulnerabilities.",
- "type": "array",
- "items": {
- "type": "object",
- "description": "Informational flags identified and assigned to a vulnerability.",
- "required": [
- "type",
- "origin",
- "description"
- ],
- "properties": {
- "type": {
- "type": "string",
- "minLength": 1,
- "description": "Result of the scan.",
- "enum": [
- "flagged-as-likely-false-positive"
- ]
- },
- "origin": {
- "minLength": 1,
- "description": "Tool that issued the flag.",
- "type": "string"
- },
- "description": {
- "minLength": 1,
- "description": "What the flag is about.",
- "type": "string"
- }
- }
- }
- },
- "location": {
- "description": "The location of the error",
- "type": "object",
- "properties": {
- "crash_address": {
- "type": "string",
- "description": "The relative address in memory were the crash occurred.",
- "examples": [
- "0xabababab"
- ]
- },
- "stacktrace_snippet": {
- "type": "string",
- "description": "The stack trace recorded during fuzzing resulting the crash.",
- "examples": [
- "func_a+0xabcd\nfunc_b+0xabcc"
- ]
- },
- "crash_state": {
- "type": "string",
- "description": "Minimised and normalized crash stack-trace (called crash_state).",
- "examples": [
- "func_a+0xa\nfunc_b+0xb\nfunc_c+0xc"
- ]
- },
- "crash_type": {
- "type": "string",
- "description": "Type of the crash.",
- "examples": [
- "Heap-Buffer-overflow",
- "Division-by-zero"
- ]
- }
- }
- }
- }
- }
- },
- "remediations": {
- "type": "array",
- "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
- "items": {
- "type": "object",
- "required": [
- "fixes",
- "summary",
- "diff"
- ],
- "properties": {
- "fixes": {
- "type": "array",
- "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
- "items": {
- "type": "object",
- "required": [
- "cve"
- ],
- "properties": {
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- }
- }
- }
- },
- "summary": {
- "type": "string",
- "minLength": 1,
- "description": "An overview of how the vulnerabilities were fixed."
- },
- "diff": {
- "type": "string",
- "minLength": 1,
- "description": "A base64-encoded remediation code diff, compatible with git apply."
- }
- }
- }
- }
- }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.4/dast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.4/dast-report-format.json
deleted file mode 100644
index b3ce7609aea..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.4/dast-report-format.json
+++ /dev/null
@@ -1,1291 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Report format for GitLab DAST",
- "description": "This schema provides the the report format for Dynamic Application Security Testing (https://docs.gitlab.com/ee/user/application_security/dast).",
- "definitions": {
- "detail_type": {
- "oneOf": [
- {
- "$ref": "#/definitions/named_list"
- },
- {
- "$ref": "#/definitions/list"
- },
- {
- "$ref": "#/definitions/table"
- },
- {
- "$ref": "#/definitions/text"
- },
- {
- "$ref": "#/definitions/url"
- },
- {
- "$ref": "#/definitions/code"
- },
- {
- "$ref": "#/definitions/value"
- },
- {
- "$ref": "#/definitions/diff"
- },
- {
- "$ref": "#/definitions/markdown"
- },
- {
- "$ref": "#/definitions/commit"
- },
- {
- "$ref": "#/definitions/file_location"
- },
- {
- "$ref": "#/definitions/module_location"
- }
- ]
- },
- "text_value": {
- "type": "string"
- },
- "named_field": {
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "$ref": "#/definitions/text_value",
- "minLength": 1
- },
- "description": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "named_list": {
- "type": "object",
- "description": "An object with named and typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "named-list"
- },
- "items": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- {
- "$ref": "#/definitions/named_field"
- },
- {
- "$ref": "#/definitions/detail_type"
- }
- ]
- }
- }
- }
- }
- },
- "list": {
- "type": "object",
- "description": "A list of typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- },
- "table": {
- "type": "object",
- "description": "A table of typed fields",
- "required": [
- "type",
- "rows"
- ],
- "properties": {
- "type": {
- "const": "table"
- },
- "header": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- },
- "rows": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- }
- },
- "text": {
- "type": "object",
- "description": "Raw text",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "text"
- },
- "value": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "url": {
- "type": "object",
- "description": "A single URL",
- "required": [
- "type",
- "href"
- ],
- "properties": {
- "type": {
- "const": "url"
- },
- "text": {
- "$ref": "#/definitions/text_value"
- },
- "href": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "http://mysite.com"
- ]
- }
- }
- },
- "code": {
- "type": "object",
- "description": "A codeblock",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "code"
- },
- "value": {
- "type": "string"
- },
- "lang": {
- "type": "string",
- "description": "A programming language"
- }
- }
- },
- "value": {
- "type": "object",
- "description": "A field that can store a range of types of value",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "value"
- },
- "value": {
- "type": [
- "number",
- "string",
- "boolean"
- ]
- }
- }
- },
- "diff": {
- "type": "object",
- "description": "A diff",
- "required": [
- "type",
- "before",
- "after"
- ],
- "properties": {
- "type": {
- "const": "diff"
- },
- "before": {
- "type": "string"
- },
- "after": {
- "type": "string"
- }
- }
- },
- "markdown": {
- "type": "object",
- "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "markdown"
- },
- "value": {
- "$ref": "#/definitions/text_value",
- "examples": [
- "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
- ]
- }
- }
- },
- "commit": {
- "type": "object",
- "description": "A commit/tag/branch within the GitLab project",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "commit"
- },
- "value": {
- "type": "string",
- "description": "The commit SHA",
- "minLength": 1
- }
- }
- },
- "file_location": {
- "type": "object",
- "description": "A location within a file in the project",
- "required": [
- "type",
- "file_name",
- "line_start"
- ],
- "properties": {
- "type": {
- "const": "file-location"
- },
- "file_name": {
- "type": "string",
- "minLength": 1
- },
- "line_start": {
- "type": "integer"
- },
- "line_end": {
- "type": "integer"
- }
- }
- },
- "module_location": {
- "type": "object",
- "description": "A location within a binary module of the form module+relative_offset",
- "required": [
- "type",
- "module_name",
- "offset"
- ],
- "properties": {
- "type": {
- "const": "module-location"
- },
- "module_name": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "compiled_binary"
- ]
- },
- "offset": {
- "type": "integer",
- "examples": [
- 100
- ]
- }
- }
- }
- },
- "self": {
- "version": "14.0.4"
- },
- "required": [
- "version",
- "vulnerabilities"
- ],
- "additionalProperties": true,
- "properties": {
- "scan": {
- "type": "object",
- "required": [
- "end_time",
- "scanned_resources",
- "scanner",
- "start_time",
- "status",
- "type"
- ],
- "properties": {
- "end_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-01-28T03:26:02"
- ]
- },
- "messages": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Communication intended for the initiator of a scan.",
- "required": [
- "level",
- "value"
- ],
- "properties": {
- "level": {
- "type": "string",
- "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
- "enum": [
- "info",
- "warn",
- "fatal"
- ],
- "examples": [
- "info"
- ]
- },
- "value": {
- "type": "string",
- "description": "The message to communicate.",
- "minLength": 1,
- "examples": [
- "Permission denied, scanning aborted"
- ]
- }
- }
- }
- },
- "analyzer": {
- "type": "object",
- "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the analyzer.",
- "minLength": 1,
- "examples": [
- "gitlab-dast"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the analyzer, not required to be unique.",
- "minLength": 1,
- "examples": [
- "GitLab DAST"
- ]
- },
- "url": {
- "type": "string",
- "format": "uri",
- "pattern": "^https?://.+",
- "description": "A link to more information about the analyzer.",
- "examples": [
- "https://docs.gitlab.com/ee/user/application_security/dast"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the analyzer.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- },
- "version": {
- "type": "string",
- "description": "The version of the analyzer.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- }
- }
- },
- "scanner": {
- "type": "object",
- "description": "Object defining the scanner used to perform the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the scanner.",
- "minLength": 1,
- "examples": [
- "my-sast-scanner"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the scanner, not required to be unique.",
- "minLength": 1,
- "examples": [
- "My SAST Scanner"
- ]
- },
- "url": {
- "type": "string",
- "description": "A link to more information about the scanner.",
- "examples": [
- "https://scanner.url"
- ]
- },
- "version": {
- "type": "string",
- "description": "The version of the scanner.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the scanner.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- }
- }
- },
- "start_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-02-14T16:01:59"
- ]
- },
- "status": {
- "type": "string",
- "description": "Result of the scan.",
- "enum": [
- "success",
- "failure"
- ]
- },
- "type": {
- "type": "string",
- "description": "Type of the scan.",
- "enum": [
- "dast",
- "api_fuzzing"
- ]
- },
- "scanned_resources": {
- "type": "array",
- "description": "The attack surface scanned by DAST.",
- "items": {
- "type": "object",
- "required": [
- "method",
- "url",
- "type"
- ],
- "properties": {
- "method": {
- "type": "string",
- "minLength": 1,
- "description": "HTTP method of the scanned resource.",
- "examples": [
- "GET",
- "POST",
- "HEAD"
- ]
- },
- "url": {
- "type": "string",
- "minLength": 1,
- "description": "URL of the scanned resource.",
- "examples": [
- "http://my.site.com/a-page"
- ]
- },
- "type": {
- "type": "string",
- "minLength": 1,
- "description": "Type of the scanned resource, for DAST, this must be 'url'.",
- "examples": [
- "url"
- ]
- }
- }
- }
- }
- }
- },
- "schema": {
- "type": "string",
- "description": "URI pointing to the validating security report schema.",
- "format": "uri"
- },
- "version": {
- "type": "string",
- "description": "The version of the schema to which the JSON report conforms.",
- "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
- },
- "vulnerabilities": {
- "type": "array",
- "description": "Array of vulnerability objects.",
- "items": {
- "type": "object",
- "description": "Describes the vulnerability.",
- "required": [
- "category",
- "cve",
- "identifiers",
- "location",
- "scanner"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
- "examples": [
- "642735a5-1425-428d-8d4e-3c854885a3c9"
- ]
- },
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
- "name": {
- "type": "string",
- "description": "The name of the vulnerability. This must not include the finding's specific information."
- },
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
- "description": {
- "type": "string",
- "description": "A long text section describing the vulnerability more fully."
- },
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
- "severity": {
- "type": "string",
- "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Info",
- "Unknown",
- "Low",
- "Medium",
- "High",
- "Critical"
- ]
- },
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
- "solution": {
- "type": "string",
- "description": "Explanation of how to fix the vulnerability."
- },
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
- "identifiers": {
- "type": "array",
- "minItems": 1,
- "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
- "items": {
- "type": "object",
- "required": [
- "type",
- "name",
- "value"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
- "minLength": 1
- },
- "name": {
- "type": "string",
- "description": "Human-readable name of the identifier.",
- "minLength": 1
- },
- "url": {
- "type": "string",
- "description": "URL of the identifier's documentation.",
- "format": "uri"
- },
- "value": {
- "type": "string",
- "description": "Value of the identifier, for matching purpose.",
- "minLength": 1
- }
- }
- }
- },
- "links": {
- "type": "array",
- "description": "An array of references to external documentation or articles that describe the vulnerability.",
- "items": {
- "type": "object",
- "required": [
- "url"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the vulnerability details link."
- },
- "url": {
- "type": "string",
- "description": "URL of the vulnerability details document.",
- "format": "uri"
- }
- }
- }
- },
- "details": {
- "$ref": "#/definitions/named_list/properties/items"
- },
- "tracking": {
- "description": "Describes how this vulnerability should be tracked as the project changes.",
- "oneOf": [
- {
- "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
- "required": [
- "items"
- ],
- "properties": {
- "type": {
- "const": "source"
- },
- "items": {
- "type": "array",
- "items": {
- "description": "An item that should be tracked using source-specific tracking methods.",
- "type": "object",
- "required": [
- "signatures"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the file that includes the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the file that includes the vulnerability."
- },
- "signatures": {
- "type": "array",
- "description": "An array of calculated tracking signatures for this tracking item.",
- "minItems": 1,
- "items": {
- "description": "A calculated tracking signature value and metadata.",
- "required": [
- "algorithm",
- "value"
- ],
- "properties": {
- "algorithm": {
- "type": "string",
- "description": "The algorithm used to generate the signature."
- },
- "value": {
- "type": "string",
- "description": "The result of this signature algorithm."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "Each tracking type must declare its own type."
- }
- }
- },
- "flags": {
- "description": "Flags that can be attached to vulnerabilities.",
- "type": "array",
- "items": {
- "type": "object",
- "description": "Informational flags identified and assigned to a vulnerability.",
- "required": [
- "type",
- "origin",
- "description"
- ],
- "properties": {
- "type": {
- "type": "string",
- "minLength": 1,
- "description": "Result of the scan.",
- "enum": [
- "flagged-as-likely-false-positive"
- ]
- },
- "origin": {
- "minLength": 1,
- "description": "Tool that issued the flag.",
- "type": "string"
- },
- "description": {
- "minLength": 1,
- "description": "What the flag is about.",
- "type": "string"
- }
- }
- }
- },
- "evidence": {
- "type": "object",
- "properties": {
- "source": {
- "type": "object",
- "description": "Source of evidence",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "Unique source identifier",
- "examples": [
- "assert:LogAnalysis",
- "assert:StatusCode"
- ]
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Source display name",
- "examples": [
- "Log Analysis",
- "Status Code"
- ]
- },
- "url": {
- "type": "string",
- "description": "Link to additional information",
- "examples": [
- "https://docs.gitlab.com/ee/development/integrations/secure.html"
- ]
- }
- }
- },
- "summary": {
- "type": "string",
- "description": "Human readable string containing evidence of the vulnerability.",
- "examples": [
- "Credit card 4111111111111111 found",
- "Server leaked information nginx/1.17.6"
- ]
- },
- "request": {
- "type": "object",
- "description": "An HTTP request.",
- "required": [
- "headers",
- "method",
- "url"
- ],
- "properties": {
- "headers": {
- "type": "array",
- "description": "HTTP headers present on the request.",
- "items": {
- "type": "object",
- "required": [
- "name",
- "value"
- ],
- "properties": {
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Name of the HTTP header.",
- "examples": [
- "Accept",
- "Content-Length",
- "Content-Type"
- ]
- },
- "value": {
- "type": "string",
- "minLength": 1,
- "description": "Value of the HTTP header.",
- "examples": [
- "*/*",
- "560",
- "application/json; charset=utf-8"
- ]
- }
- }
- }
- },
- "method": {
- "type": "string",
- "minLength": 1,
- "description": "HTTP method used in the request.",
- "examples": [
- "GET",
- "POST"
- ]
- },
- "url": {
- "type": "string",
- "minLength": 1,
- "description": "URL of the request.",
- "examples": [
- "http://my.site.com/vulnerable-endpoint?show-credit-card"
- ]
- },
- "body": {
- "type": "string",
- "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.",
- "examples": [
- "user=jsmith&first=%27&last=smith"
- ]
- }
- }
- },
- "response": {
- "type": "object",
- "description": "An HTTP response.",
- "required": [
- "headers",
- "reason_phrase",
- "status_code"
- ],
- "properties": {
- "headers": {
- "type": "array",
- "description": "HTTP headers present on the request.",
- "items": {
- "type": "object",
- "required": [
- "name",
- "value"
- ],
- "properties": {
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Name of the HTTP header.",
- "examples": [
- "Accept",
- "Content-Length",
- "Content-Type"
- ]
- },
- "value": {
- "type": "string",
- "minLength": 1,
- "description": "Value of the HTTP header.",
- "examples": [
- "*/*",
- "560",
- "application/json; charset=utf-8"
- ]
- }
- }
- }
- },
- "reason_phrase": {
- "type": "string",
- "description": "HTTP reason phrase of the response.",
- "examples": [
- "OK",
- "Internal Server Error"
- ]
- },
- "status_code": {
- "type": "integer",
- "description": "HTTP status code of the response.",
- "examples": [
- 200,
- 500
- ]
- },
- "body": {
- "type": "string",
- "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.",
- "examples": [
- "{\"user_id\": 2}"
- ]
- }
- }
- },
- "supporting_messages": {
- "type": "array",
- "description": "Array of supporting http messages.",
- "items": {
- "type": "object",
- "description": "A supporting http message.",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Message display name.",
- "examples": [
- "Unmodified",
- "Recorded"
- ]
- },
- "request": {
- "type": "object",
- "description": "An HTTP request.",
- "required": [
- "headers",
- "method",
- "url"
- ],
- "properties": {
- "headers": {
- "type": "array",
- "description": "HTTP headers present on the request.",
- "items": {
- "type": "object",
- "required": [
- "name",
- "value"
- ],
- "properties": {
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Name of the HTTP header.",
- "examples": [
- "Accept",
- "Content-Length",
- "Content-Type"
- ]
- },
- "value": {
- "type": "string",
- "minLength": 1,
- "description": "Value of the HTTP header.",
- "examples": [
- "*/*",
- "560",
- "application/json; charset=utf-8"
- ]
- }
- }
- }
- },
- "method": {
- "type": "string",
- "minLength": 1,
- "description": "HTTP method used in the request.",
- "examples": [
- "GET",
- "POST"
- ]
- },
- "url": {
- "type": "string",
- "minLength": 1,
- "description": "URL of the request.",
- "examples": [
- "http://my.site.com/vulnerable-endpoint?show-credit-card"
- ]
- },
- "body": {
- "type": "string",
- "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.",
- "examples": [
- "user=jsmith&first=%27&last=smith"
- ]
- }
- }
- },
- "response": {
- "type": "object",
- "description": "An HTTP response.",
- "required": [
- "headers",
- "reason_phrase",
- "status_code"
- ],
- "properties": {
- "headers": {
- "type": "array",
- "description": "HTTP headers present on the request.",
- "items": {
- "type": "object",
- "required": [
- "name",
- "value"
- ],
- "properties": {
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Name of the HTTP header.",
- "examples": [
- "Accept",
- "Content-Length",
- "Content-Type"
- ]
- },
- "value": {
- "type": "string",
- "minLength": 1,
- "description": "Value of the HTTP header.",
- "examples": [
- "*/*",
- "560",
- "application/json; charset=utf-8"
- ]
- }
- }
- }
- },
- "reason_phrase": {
- "type": "string",
- "description": "HTTP reason phrase of the response.",
- "examples": [
- "OK",
- "Internal Server Error"
- ]
- },
- "status_code": {
- "type": "integer",
- "description": "HTTP status code of the response.",
- "examples": [
- 200,
- 500
- ]
- },
- "body": {
- "type": "string",
- "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.",
- "examples": [
- "{\"user_id\": 2}"
- ]
- }
- }
- }
- }
- }
- }
- }
- },
- "location": {
- "type": "object",
- "description": "Identifies the vulnerability's location.",
- "properties": {
- "hostname": {
- "type": "string",
- "description": "The protocol, domain, and port of the application where the vulnerability was found."
- },
- "method": {
- "type": "string",
- "description": "The HTTP method that was used to request the URL where the vulnerability was found."
- },
- "param": {
- "type": "string",
- "description": "A value provided by a vulnerability rule related to the found vulnerability. Examples include a header value, or a parameter used in a HTTP POST."
- },
- "path": {
- "type": "string",
- "description": "The path of the URL where the vulnerability was found. Typically, this would start with a forward slash."
- }
- }
- },
- "assets": {
- "type": "array",
- "description": "Array of build assets associated with vulnerability.",
- "items": {
- "type": "object",
- "description": "Describes an asset associated with vulnerability.",
- "required": [
- "type",
- "name",
- "url"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "The type of asset",
- "enum": [
- "http_session",
- "postman"
- ]
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Display name for asset",
- "examples": [
- "HTTP Messages",
- "Postman Collection"
- ]
- },
- "url": {
- "type": "string",
- "minLength": 1,
- "description": "Link to asset in build artifacts",
- "examples": [
- "https://gitlab.com/gitlab-org/security-products/dast/-/jobs/626397001/artifacts/file//output/zap_session.data"
- ]
- }
- }
- }
- },
- "discovered_at": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss.sss, representing when the vulnerability was discovered",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}\\.\\d{3}$",
- "examples": [
- "2020-01-28T03:26:02.956"
- ]
- }
- }
- }
- },
- "remediations": {
- "type": "array",
- "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
- "items": {
- "type": "object",
- "required": [
- "fixes",
- "summary",
- "diff"
- ],
- "properties": {
- "fixes": {
- "type": "array",
- "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
- "items": {
- "type": "object",
- "required": [
- "cve"
- ],
- "properties": {
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- }
- }
- }
- },
- "summary": {
- "type": "string",
- "minLength": 1,
- "description": "An overview of how the vulnerabilities were fixed."
- },
- "diff": {
- "type": "string",
- "minLength": 1,
- "description": "A base64-encoded remediation code diff, compatible with git apply."
- }
- }
- }
- }
- }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.4/dependency-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.4/dependency-scanning-report-format.json
deleted file mode 100644
index 605d379e497..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.4/dependency-scanning-report-format.json
+++ /dev/null
@@ -1,968 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Report format for GitLab Dependency Scanning",
- "description": "This schema provides the the report format for Dependency Scanning analyzers (https://docs.gitlab.com/ee/user/application_security/dependency_scanning).",
- "definitions": {
- "detail_type": {
- "oneOf": [
- {
- "$ref": "#/definitions/named_list"
- },
- {
- "$ref": "#/definitions/list"
- },
- {
- "$ref": "#/definitions/table"
- },
- {
- "$ref": "#/definitions/text"
- },
- {
- "$ref": "#/definitions/url"
- },
- {
- "$ref": "#/definitions/code"
- },
- {
- "$ref": "#/definitions/value"
- },
- {
- "$ref": "#/definitions/diff"
- },
- {
- "$ref": "#/definitions/markdown"
- },
- {
- "$ref": "#/definitions/commit"
- },
- {
- "$ref": "#/definitions/file_location"
- },
- {
- "$ref": "#/definitions/module_location"
- }
- ]
- },
- "text_value": {
- "type": "string"
- },
- "named_field": {
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "$ref": "#/definitions/text_value",
- "minLength": 1
- },
- "description": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "named_list": {
- "type": "object",
- "description": "An object with named and typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "named-list"
- },
- "items": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- {
- "$ref": "#/definitions/named_field"
- },
- {
- "$ref": "#/definitions/detail_type"
- }
- ]
- }
- }
- }
- }
- },
- "list": {
- "type": "object",
- "description": "A list of typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- },
- "table": {
- "type": "object",
- "description": "A table of typed fields",
- "required": [
- "type",
- "rows"
- ],
- "properties": {
- "type": {
- "const": "table"
- },
- "header": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- },
- "rows": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- }
- },
- "text": {
- "type": "object",
- "description": "Raw text",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "text"
- },
- "value": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "url": {
- "type": "object",
- "description": "A single URL",
- "required": [
- "type",
- "href"
- ],
- "properties": {
- "type": {
- "const": "url"
- },
- "text": {
- "$ref": "#/definitions/text_value"
- },
- "href": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "http://mysite.com"
- ]
- }
- }
- },
- "code": {
- "type": "object",
- "description": "A codeblock",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "code"
- },
- "value": {
- "type": "string"
- },
- "lang": {
- "type": "string",
- "description": "A programming language"
- }
- }
- },
- "value": {
- "type": "object",
- "description": "A field that can store a range of types of value",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "value"
- },
- "value": {
- "type": [
- "number",
- "string",
- "boolean"
- ]
- }
- }
- },
- "diff": {
- "type": "object",
- "description": "A diff",
- "required": [
- "type",
- "before",
- "after"
- ],
- "properties": {
- "type": {
- "const": "diff"
- },
- "before": {
- "type": "string"
- },
- "after": {
- "type": "string"
- }
- }
- },
- "markdown": {
- "type": "object",
- "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "markdown"
- },
- "value": {
- "$ref": "#/definitions/text_value",
- "examples": [
- "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
- ]
- }
- }
- },
- "commit": {
- "type": "object",
- "description": "A commit/tag/branch within the GitLab project",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "commit"
- },
- "value": {
- "type": "string",
- "description": "The commit SHA",
- "minLength": 1
- }
- }
- },
- "file_location": {
- "type": "object",
- "description": "A location within a file in the project",
- "required": [
- "type",
- "file_name",
- "line_start"
- ],
- "properties": {
- "type": {
- "const": "file-location"
- },
- "file_name": {
- "type": "string",
- "minLength": 1
- },
- "line_start": {
- "type": "integer"
- },
- "line_end": {
- "type": "integer"
- }
- }
- },
- "module_location": {
- "type": "object",
- "description": "A location within a binary module of the form module+relative_offset",
- "required": [
- "type",
- "module_name",
- "offset"
- ],
- "properties": {
- "type": {
- "const": "module-location"
- },
- "module_name": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "compiled_binary"
- ]
- },
- "offset": {
- "type": "integer",
- "examples": [
- 100
- ]
- }
- }
- }
- },
- "self": {
- "version": "14.0.4"
- },
- "required": [
- "dependency_files",
- "version",
- "vulnerabilities"
- ],
- "additionalProperties": true,
- "properties": {
- "scan": {
- "type": "object",
- "required": [
- "end_time",
- "scanner",
- "start_time",
- "status",
- "type"
- ],
- "properties": {
- "end_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-01-28T03:26:02"
- ]
- },
- "messages": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Communication intended for the initiator of a scan.",
- "required": [
- "level",
- "value"
- ],
- "properties": {
- "level": {
- "type": "string",
- "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
- "enum": [
- "info",
- "warn",
- "fatal"
- ],
- "examples": [
- "info"
- ]
- },
- "value": {
- "type": "string",
- "description": "The message to communicate.",
- "minLength": 1,
- "examples": [
- "Permission denied, scanning aborted"
- ]
- }
- }
- }
- },
- "analyzer": {
- "type": "object",
- "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the analyzer.",
- "minLength": 1,
- "examples": [
- "gitlab-dast"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the analyzer, not required to be unique.",
- "minLength": 1,
- "examples": [
- "GitLab DAST"
- ]
- },
- "url": {
- "type": "string",
- "format": "uri",
- "pattern": "^https?://.+",
- "description": "A link to more information about the analyzer.",
- "examples": [
- "https://docs.gitlab.com/ee/user/application_security/dast"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the analyzer.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- },
- "version": {
- "type": "string",
- "description": "The version of the analyzer.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- }
- }
- },
- "scanner": {
- "type": "object",
- "description": "Object defining the scanner used to perform the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the scanner.",
- "minLength": 1,
- "examples": [
- "my-sast-scanner"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the scanner, not required to be unique.",
- "minLength": 1,
- "examples": [
- "My SAST Scanner"
- ]
- },
- "url": {
- "type": "string",
- "description": "A link to more information about the scanner.",
- "examples": [
- "https://scanner.url"
- ]
- },
- "version": {
- "type": "string",
- "description": "The version of the scanner.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the scanner.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- }
- }
- },
- "start_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-02-14T16:01:59"
- ]
- },
- "status": {
- "type": "string",
- "description": "Result of the scan.",
- "enum": [
- "success",
- "failure"
- ]
- },
- "type": {
- "type": "string",
- "description": "Type of the scan.",
- "enum": [
- "dependency_scanning"
- ]
- }
- }
- },
- "schema": {
- "type": "string",
- "description": "URI pointing to the validating security report schema.",
- "format": "uri"
- },
- "version": {
- "type": "string",
- "description": "The version of the schema to which the JSON report conforms.",
- "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
- },
- "vulnerabilities": {
- "type": "array",
- "description": "Array of vulnerability objects.",
- "items": {
- "type": "object",
- "description": "Describes the vulnerability.",
- "required": [
- "category",
- "cve",
- "identifiers",
- "location",
- "scanner"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
- "examples": [
- "642735a5-1425-428d-8d4e-3c854885a3c9"
- ]
- },
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
- "name": {
- "type": "string",
- "description": "The name of the vulnerability. This must not include the finding's specific information."
- },
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
- "description": {
- "type": "string",
- "description": "A long text section describing the vulnerability more fully."
- },
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
- "severity": {
- "type": "string",
- "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Info",
- "Unknown",
- "Low",
- "Medium",
- "High",
- "Critical"
- ]
- },
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
- "solution": {
- "type": "string",
- "description": "Explanation of how to fix the vulnerability."
- },
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
- "identifiers": {
- "type": "array",
- "minItems": 1,
- "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
- "items": {
- "type": "object",
- "required": [
- "type",
- "name",
- "value"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
- "minLength": 1
- },
- "name": {
- "type": "string",
- "description": "Human-readable name of the identifier.",
- "minLength": 1
- },
- "url": {
- "type": "string",
- "description": "URL of the identifier's documentation.",
- "format": "uri"
- },
- "value": {
- "type": "string",
- "description": "Value of the identifier, for matching purpose.",
- "minLength": 1
- }
- }
- }
- },
- "links": {
- "type": "array",
- "description": "An array of references to external documentation or articles that describe the vulnerability.",
- "items": {
- "type": "object",
- "required": [
- "url"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the vulnerability details link."
- },
- "url": {
- "type": "string",
- "description": "URL of the vulnerability details document.",
- "format": "uri"
- }
- }
- }
- },
- "details": {
- "$ref": "#/definitions/named_list/properties/items"
- },
- "tracking": {
- "description": "Describes how this vulnerability should be tracked as the project changes.",
- "oneOf": [
- {
- "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
- "required": [
- "items"
- ],
- "properties": {
- "type": {
- "const": "source"
- },
- "items": {
- "type": "array",
- "items": {
- "description": "An item that should be tracked using source-specific tracking methods.",
- "type": "object",
- "required": [
- "signatures"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the file that includes the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the file that includes the vulnerability."
- },
- "signatures": {
- "type": "array",
- "description": "An array of calculated tracking signatures for this tracking item.",
- "minItems": 1,
- "items": {
- "description": "A calculated tracking signature value and metadata.",
- "required": [
- "algorithm",
- "value"
- ],
- "properties": {
- "algorithm": {
- "type": "string",
- "description": "The algorithm used to generate the signature."
- },
- "value": {
- "type": "string",
- "description": "The result of this signature algorithm."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "Each tracking type must declare its own type."
- }
- }
- },
- "flags": {
- "description": "Flags that can be attached to vulnerabilities.",
- "type": "array",
- "items": {
- "type": "object",
- "description": "Informational flags identified and assigned to a vulnerability.",
- "required": [
- "type",
- "origin",
- "description"
- ],
- "properties": {
- "type": {
- "type": "string",
- "minLength": 1,
- "description": "Result of the scan.",
- "enum": [
- "flagged-as-likely-false-positive"
- ]
- },
- "origin": {
- "minLength": 1,
- "description": "Tool that issued the flag.",
- "type": "string"
- },
- "description": {
- "minLength": 1,
- "description": "What the flag is about.",
- "type": "string"
- }
- }
- }
- },
- "location": {
- "type": "object",
- "description": "Identifies the vulnerability's location.",
- "required": [
- "file",
- "dependency"
- ],
- "properties": {
- "file": {
- "type": "string",
- "minLength": 1,
- "description": "Path to the manifest or lock file where the dependency is declared (such as yarn.lock)."
- },
- "dependency": {
- "type": "object",
- "description": "Describes the dependency of a project where the vulnerability is located.",
- "properties": {
- "package": {
- "type": "object",
- "description": "Provides information on the package where the vulnerability is located.",
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the package where the vulnerability is located."
- }
- }
- },
- "version": {
- "type": "string",
- "description": "Version of the vulnerable package."
- },
- "iid": {
- "description": "ID that identifies the dependency in the scope of a dependency file.",
- "type": "number"
- },
- "direct": {
- "type": "boolean",
- "description": "Tells whether this is a direct, top-level dependency of the scanned project."
- },
- "dependency_path": {
- "type": "array",
- "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.",
- "items": {
- "type": "object",
- "required": [
- "iid"
- ],
- "properties": {
- "iid": {
- "type": "number",
- "description": "ID that is unique in the scope of a parent object, and specific to the resource type."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- }
- },
- "remediations": {
- "type": "array",
- "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
- "items": {
- "type": "object",
- "required": [
- "fixes",
- "summary",
- "diff"
- ],
- "properties": {
- "fixes": {
- "type": "array",
- "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
- "items": {
- "type": "object",
- "required": [
- "cve"
- ],
- "properties": {
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- }
- }
- }
- },
- "summary": {
- "type": "string",
- "minLength": 1,
- "description": "An overview of how the vulnerabilities were fixed."
- },
- "diff": {
- "type": "string",
- "minLength": 1,
- "description": "A base64-encoded remediation code diff, compatible with git apply."
- }
- }
- }
- },
- "dependency_files": {
- "type": "array",
- "description": "List of dependency files identified in the project.",
- "items": {
- "type": "object",
- "required": [
- "path",
- "package_manager",
- "dependencies"
- ],
- "properties": {
- "path": {
- "type": "string",
- "minLength": 1
- },
- "package_manager": {
- "type": "string",
- "minLength": 1
- },
- "dependencies": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Describes the dependency of a project where the vulnerability is located.",
- "properties": {
- "package": {
- "type": "object",
- "description": "Provides information on the package where the vulnerability is located.",
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the package where the vulnerability is located."
- }
- }
- },
- "version": {
- "type": "string",
- "description": "Version of the vulnerable package."
- },
- "iid": {
- "description": "ID that identifies the dependency in the scope of a dependency file.",
- "type": "number"
- },
- "direct": {
- "type": "boolean",
- "description": "Tells whether this is a direct, top-level dependency of the scanned project."
- },
- "dependency_path": {
- "type": "array",
- "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.",
- "items": {
- "type": "object",
- "required": [
- "iid"
- ],
- "properties": {
- "iid": {
- "type": "number",
- "description": "ID that is unique in the scope of a parent object, and specific to the resource type."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- }
- }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.4/sast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.4/sast-report-format.json
deleted file mode 100644
index 2d9e1af6663..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.4/sast-report-format.json
+++ /dev/null
@@ -1,869 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Report format for GitLab SAST",
- "description": "This schema provides the report format for Static Application Security Testing analyzers (https://docs.gitlab.com/ee/user/application_security/sast).",
- "definitions": {
- "detail_type": {
- "oneOf": [
- {
- "$ref": "#/definitions/named_list"
- },
- {
- "$ref": "#/definitions/list"
- },
- {
- "$ref": "#/definitions/table"
- },
- {
- "$ref": "#/definitions/text"
- },
- {
- "$ref": "#/definitions/url"
- },
- {
- "$ref": "#/definitions/code"
- },
- {
- "$ref": "#/definitions/value"
- },
- {
- "$ref": "#/definitions/diff"
- },
- {
- "$ref": "#/definitions/markdown"
- },
- {
- "$ref": "#/definitions/commit"
- },
- {
- "$ref": "#/definitions/file_location"
- },
- {
- "$ref": "#/definitions/module_location"
- }
- ]
- },
- "text_value": {
- "type": "string"
- },
- "named_field": {
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "$ref": "#/definitions/text_value",
- "minLength": 1
- },
- "description": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "named_list": {
- "type": "object",
- "description": "An object with named and typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "named-list"
- },
- "items": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- {
- "$ref": "#/definitions/named_field"
- },
- {
- "$ref": "#/definitions/detail_type"
- }
- ]
- }
- }
- }
- }
- },
- "list": {
- "type": "object",
- "description": "A list of typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- },
- "table": {
- "type": "object",
- "description": "A table of typed fields",
- "required": [
- "type",
- "rows"
- ],
- "properties": {
- "type": {
- "const": "table"
- },
- "header": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- },
- "rows": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- }
- },
- "text": {
- "type": "object",
- "description": "Raw text",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "text"
- },
- "value": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "url": {
- "type": "object",
- "description": "A single URL",
- "required": [
- "type",
- "href"
- ],
- "properties": {
- "type": {
- "const": "url"
- },
- "text": {
- "$ref": "#/definitions/text_value"
- },
- "href": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "http://mysite.com"
- ]
- }
- }
- },
- "code": {
- "type": "object",
- "description": "A codeblock",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "code"
- },
- "value": {
- "type": "string"
- },
- "lang": {
- "type": "string",
- "description": "A programming language"
- }
- }
- },
- "value": {
- "type": "object",
- "description": "A field that can store a range of types of value",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "value"
- },
- "value": {
- "type": [
- "number",
- "string",
- "boolean"
- ]
- }
- }
- },
- "diff": {
- "type": "object",
- "description": "A diff",
- "required": [
- "type",
- "before",
- "after"
- ],
- "properties": {
- "type": {
- "const": "diff"
- },
- "before": {
- "type": "string"
- },
- "after": {
- "type": "string"
- }
- }
- },
- "markdown": {
- "type": "object",
- "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "markdown"
- },
- "value": {
- "$ref": "#/definitions/text_value",
- "examples": [
- "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
- ]
- }
- }
- },
- "commit": {
- "type": "object",
- "description": "A commit/tag/branch within the GitLab project",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "commit"
- },
- "value": {
- "type": "string",
- "description": "The commit SHA",
- "minLength": 1
- }
- }
- },
- "file_location": {
- "type": "object",
- "description": "A location within a file in the project",
- "required": [
- "type",
- "file_name",
- "line_start"
- ],
- "properties": {
- "type": {
- "const": "file-location"
- },
- "file_name": {
- "type": "string",
- "minLength": 1
- },
- "line_start": {
- "type": "integer"
- },
- "line_end": {
- "type": "integer"
- }
- }
- },
- "module_location": {
- "type": "object",
- "description": "A location within a binary module of the form module+relative_offset",
- "required": [
- "type",
- "module_name",
- "offset"
- ],
- "properties": {
- "type": {
- "const": "module-location"
- },
- "module_name": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "compiled_binary"
- ]
- },
- "offset": {
- "type": "integer",
- "examples": [
- 100
- ]
- }
- }
- }
- },
- "self": {
- "version": "14.0.4"
- },
- "required": [
- "version",
- "vulnerabilities"
- ],
- "additionalProperties": true,
- "properties": {
- "scan": {
- "type": "object",
- "required": [
- "end_time",
- "scanner",
- "start_time",
- "status",
- "type"
- ],
- "properties": {
- "end_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-01-28T03:26:02"
- ]
- },
- "messages": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Communication intended for the initiator of a scan.",
- "required": [
- "level",
- "value"
- ],
- "properties": {
- "level": {
- "type": "string",
- "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
- "enum": [
- "info",
- "warn",
- "fatal"
- ],
- "examples": [
- "info"
- ]
- },
- "value": {
- "type": "string",
- "description": "The message to communicate.",
- "minLength": 1,
- "examples": [
- "Permission denied, scanning aborted"
- ]
- }
- }
- }
- },
- "analyzer": {
- "type": "object",
- "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the analyzer.",
- "minLength": 1,
- "examples": [
- "gitlab-dast"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the analyzer, not required to be unique.",
- "minLength": 1,
- "examples": [
- "GitLab DAST"
- ]
- },
- "url": {
- "type": "string",
- "format": "uri",
- "pattern": "^https?://.+",
- "description": "A link to more information about the analyzer.",
- "examples": [
- "https://docs.gitlab.com/ee/user/application_security/dast"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the analyzer.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- },
- "version": {
- "type": "string",
- "description": "The version of the analyzer.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- }
- }
- },
- "scanner": {
- "type": "object",
- "description": "Object defining the scanner used to perform the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the scanner.",
- "minLength": 1,
- "examples": [
- "my-sast-scanner"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the scanner, not required to be unique.",
- "minLength": 1,
- "examples": [
- "My SAST Scanner"
- ]
- },
- "url": {
- "type": "string",
- "description": "A link to more information about the scanner.",
- "examples": [
- "https://scanner.url"
- ]
- },
- "version": {
- "type": "string",
- "description": "The version of the scanner.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the scanner.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- }
- }
- },
- "start_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-02-14T16:01:59"
- ]
- },
- "status": {
- "type": "string",
- "description": "Result of the scan.",
- "enum": [
- "success",
- "failure"
- ]
- },
- "type": {
- "type": "string",
- "description": "Type of the scan.",
- "enum": [
- "sast"
- ]
- }
- }
- },
- "schema": {
- "type": "string",
- "description": "URI pointing to the validating security report schema.",
- "format": "uri"
- },
- "version": {
- "type": "string",
- "description": "The version of the schema to which the JSON report conforms.",
- "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
- },
- "vulnerabilities": {
- "type": "array",
- "description": "Array of vulnerability objects.",
- "items": {
- "type": "object",
- "description": "Describes the vulnerability.",
- "required": [
- "category",
- "cve",
- "identifiers",
- "location",
- "scanner"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
- "examples": [
- "642735a5-1425-428d-8d4e-3c854885a3c9"
- ]
- },
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
- "name": {
- "type": "string",
- "description": "The name of the vulnerability. This must not include the finding's specific information."
- },
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
- "description": {
- "type": "string",
- "description": "A long text section describing the vulnerability more fully."
- },
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
- "severity": {
- "type": "string",
- "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Info",
- "Unknown",
- "Low",
- "Medium",
- "High",
- "Critical"
- ]
- },
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
- "solution": {
- "type": "string",
- "description": "Explanation of how to fix the vulnerability."
- },
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
- "identifiers": {
- "type": "array",
- "minItems": 1,
- "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
- "items": {
- "type": "object",
- "required": [
- "type",
- "name",
- "value"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
- "minLength": 1
- },
- "name": {
- "type": "string",
- "description": "Human-readable name of the identifier.",
- "minLength": 1
- },
- "url": {
- "type": "string",
- "description": "URL of the identifier's documentation.",
- "format": "uri"
- },
- "value": {
- "type": "string",
- "description": "Value of the identifier, for matching purpose.",
- "minLength": 1
- }
- }
- }
- },
- "links": {
- "type": "array",
- "description": "An array of references to external documentation or articles that describe the vulnerability.",
- "items": {
- "type": "object",
- "required": [
- "url"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the vulnerability details link."
- },
- "url": {
- "type": "string",
- "description": "URL of the vulnerability details document.",
- "format": "uri"
- }
- }
- }
- },
- "details": {
- "$ref": "#/definitions/named_list/properties/items"
- },
- "tracking": {
- "description": "Describes how this vulnerability should be tracked as the project changes.",
- "oneOf": [
- {
- "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
- "required": [
- "items"
- ],
- "properties": {
- "type": {
- "const": "source"
- },
- "items": {
- "type": "array",
- "items": {
- "description": "An item that should be tracked using source-specific tracking methods.",
- "type": "object",
- "required": [
- "signatures"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the file that includes the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the file that includes the vulnerability."
- },
- "signatures": {
- "type": "array",
- "description": "An array of calculated tracking signatures for this tracking item.",
- "minItems": 1,
- "items": {
- "description": "A calculated tracking signature value and metadata.",
- "required": [
- "algorithm",
- "value"
- ],
- "properties": {
- "algorithm": {
- "type": "string",
- "description": "The algorithm used to generate the signature."
- },
- "value": {
- "type": "string",
- "description": "The result of this signature algorithm."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "Each tracking type must declare its own type."
- }
- }
- },
- "flags": {
- "description": "Flags that can be attached to vulnerabilities.",
- "type": "array",
- "items": {
- "type": "object",
- "description": "Informational flags identified and assigned to a vulnerability.",
- "required": [
- "type",
- "origin",
- "description"
- ],
- "properties": {
- "type": {
- "type": "string",
- "minLength": 1,
- "description": "Result of the scan.",
- "enum": [
- "flagged-as-likely-false-positive"
- ]
- },
- "origin": {
- "minLength": 1,
- "description": "Tool that issued the flag.",
- "type": "string"
- },
- "description": {
- "minLength": 1,
- "description": "What the flag is about.",
- "type": "string"
- }
- }
- }
- },
- "location": {
- "type": "object",
- "description": "Identifies the vulnerability's location.",
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the code affected by the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the code affected by the vulnerability."
- },
- "class": {
- "type": "string",
- "description": "Provides the name of the class where the vulnerability is located."
- },
- "method": {
- "type": "string",
- "description": "Provides the name of the method where the vulnerability is located."
- }
- }
- },
- "raw_source_code_extract": {
- "type": "string",
- "description": "Provides an unsanitized excerpt of the affected source code."
- }
- }
- }
- },
- "remediations": {
- "type": "array",
- "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
- "items": {
- "type": "object",
- "required": [
- "fixes",
- "summary",
- "diff"
- ],
- "properties": {
- "fixes": {
- "type": "array",
- "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
- "items": {
- "type": "object",
- "required": [
- "cve"
- ],
- "properties": {
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- }
- }
- }
- },
- "summary": {
- "type": "string",
- "minLength": 1,
- "description": "An overview of how the vulnerabilities were fixed."
- },
- "diff": {
- "type": "string",
- "minLength": 1,
- "description": "A base64-encoded remediation code diff, compatible with git apply."
- }
- }
- }
- }
- }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.4/secret-detection-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.4/secret-detection-report-format.json
deleted file mode 100644
index 70f22b243c6..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.4/secret-detection-report-format.json
+++ /dev/null
@@ -1,892 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Report format for GitLab Secret Detection",
- "description": "This schema provides the the report format for the Secret Detection analyzer (https://docs.gitlab.com/ee/user/application_security/secret_detection)",
- "definitions": {
- "detail_type": {
- "oneOf": [
- {
- "$ref": "#/definitions/named_list"
- },
- {
- "$ref": "#/definitions/list"
- },
- {
- "$ref": "#/definitions/table"
- },
- {
- "$ref": "#/definitions/text"
- },
- {
- "$ref": "#/definitions/url"
- },
- {
- "$ref": "#/definitions/code"
- },
- {
- "$ref": "#/definitions/value"
- },
- {
- "$ref": "#/definitions/diff"
- },
- {
- "$ref": "#/definitions/markdown"
- },
- {
- "$ref": "#/definitions/commit"
- },
- {
- "$ref": "#/definitions/file_location"
- },
- {
- "$ref": "#/definitions/module_location"
- }
- ]
- },
- "text_value": {
- "type": "string"
- },
- "named_field": {
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "$ref": "#/definitions/text_value",
- "minLength": 1
- },
- "description": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "named_list": {
- "type": "object",
- "description": "An object with named and typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "named-list"
- },
- "items": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- {
- "$ref": "#/definitions/named_field"
- },
- {
- "$ref": "#/definitions/detail_type"
- }
- ]
- }
- }
- }
- }
- },
- "list": {
- "type": "object",
- "description": "A list of typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- },
- "table": {
- "type": "object",
- "description": "A table of typed fields",
- "required": [
- "type",
- "rows"
- ],
- "properties": {
- "type": {
- "const": "table"
- },
- "header": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- },
- "rows": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- }
- },
- "text": {
- "type": "object",
- "description": "Raw text",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "text"
- },
- "value": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "url": {
- "type": "object",
- "description": "A single URL",
- "required": [
- "type",
- "href"
- ],
- "properties": {
- "type": {
- "const": "url"
- },
- "text": {
- "$ref": "#/definitions/text_value"
- },
- "href": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "http://mysite.com"
- ]
- }
- }
- },
- "code": {
- "type": "object",
- "description": "A codeblock",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "code"
- },
- "value": {
- "type": "string"
- },
- "lang": {
- "type": "string",
- "description": "A programming language"
- }
- }
- },
- "value": {
- "type": "object",
- "description": "A field that can store a range of types of value",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "value"
- },
- "value": {
- "type": [
- "number",
- "string",
- "boolean"
- ]
- }
- }
- },
- "diff": {
- "type": "object",
- "description": "A diff",
- "required": [
- "type",
- "before",
- "after"
- ],
- "properties": {
- "type": {
- "const": "diff"
- },
- "before": {
- "type": "string"
- },
- "after": {
- "type": "string"
- }
- }
- },
- "markdown": {
- "type": "object",
- "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "markdown"
- },
- "value": {
- "$ref": "#/definitions/text_value",
- "examples": [
- "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
- ]
- }
- }
- },
- "commit": {
- "type": "object",
- "description": "A commit/tag/branch within the GitLab project",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "commit"
- },
- "value": {
- "type": "string",
- "description": "The commit SHA",
- "minLength": 1
- }
- }
- },
- "file_location": {
- "type": "object",
- "description": "A location within a file in the project",
- "required": [
- "type",
- "file_name",
- "line_start"
- ],
- "properties": {
- "type": {
- "const": "file-location"
- },
- "file_name": {
- "type": "string",
- "minLength": 1
- },
- "line_start": {
- "type": "integer"
- },
- "line_end": {
- "type": "integer"
- }
- }
- },
- "module_location": {
- "type": "object",
- "description": "A location within a binary module of the form module+relative_offset",
- "required": [
- "type",
- "module_name",
- "offset"
- ],
- "properties": {
- "type": {
- "const": "module-location"
- },
- "module_name": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "compiled_binary"
- ]
- },
- "offset": {
- "type": "integer",
- "examples": [
- 100
- ]
- }
- }
- }
- },
- "self": {
- "version": "14.0.4"
- },
- "required": [
- "version",
- "vulnerabilities"
- ],
- "additionalProperties": true,
- "properties": {
- "scan": {
- "type": "object",
- "required": [
- "end_time",
- "scanner",
- "start_time",
- "status",
- "type"
- ],
- "properties": {
- "end_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-01-28T03:26:02"
- ]
- },
- "messages": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Communication intended for the initiator of a scan.",
- "required": [
- "level",
- "value"
- ],
- "properties": {
- "level": {
- "type": "string",
- "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
- "enum": [
- "info",
- "warn",
- "fatal"
- ],
- "examples": [
- "info"
- ]
- },
- "value": {
- "type": "string",
- "description": "The message to communicate.",
- "minLength": 1,
- "examples": [
- "Permission denied, scanning aborted"
- ]
- }
- }
- }
- },
- "analyzer": {
- "type": "object",
- "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the analyzer.",
- "minLength": 1,
- "examples": [
- "gitlab-dast"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the analyzer, not required to be unique.",
- "minLength": 1,
- "examples": [
- "GitLab DAST"
- ]
- },
- "url": {
- "type": "string",
- "format": "uri",
- "pattern": "^https?://.+",
- "description": "A link to more information about the analyzer.",
- "examples": [
- "https://docs.gitlab.com/ee/user/application_security/dast"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the analyzer.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- },
- "version": {
- "type": "string",
- "description": "The version of the analyzer.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- }
- }
- },
- "scanner": {
- "type": "object",
- "description": "Object defining the scanner used to perform the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the scanner.",
- "minLength": 1,
- "examples": [
- "my-sast-scanner"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the scanner, not required to be unique.",
- "minLength": 1,
- "examples": [
- "My SAST Scanner"
- ]
- },
- "url": {
- "type": "string",
- "description": "A link to more information about the scanner.",
- "examples": [
- "https://scanner.url"
- ]
- },
- "version": {
- "type": "string",
- "description": "The version of the scanner.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the scanner.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- }
- }
- },
- "start_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-02-14T16:01:59"
- ]
- },
- "status": {
- "type": "string",
- "description": "Result of the scan.",
- "enum": [
- "success",
- "failure"
- ]
- },
- "type": {
- "type": "string",
- "description": "Type of the scan.",
- "enum": [
- "secret_detection"
- ]
- }
- }
- },
- "schema": {
- "type": "string",
- "description": "URI pointing to the validating security report schema.",
- "format": "uri"
- },
- "version": {
- "type": "string",
- "description": "The version of the schema to which the JSON report conforms.",
- "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
- },
- "vulnerabilities": {
- "type": "array",
- "description": "Array of vulnerability objects.",
- "items": {
- "type": "object",
- "description": "Describes the vulnerability.",
- "required": [
- "category",
- "cve",
- "identifiers",
- "location",
- "scanner"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
- "examples": [
- "642735a5-1425-428d-8d4e-3c854885a3c9"
- ]
- },
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
- "name": {
- "type": "string",
- "description": "The name of the vulnerability. This must not include the finding's specific information."
- },
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
- "description": {
- "type": "string",
- "description": "A long text section describing the vulnerability more fully."
- },
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
- "severity": {
- "type": "string",
- "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Info",
- "Unknown",
- "Low",
- "Medium",
- "High",
- "Critical"
- ]
- },
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
- "solution": {
- "type": "string",
- "description": "Explanation of how to fix the vulnerability."
- },
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
- "identifiers": {
- "type": "array",
- "minItems": 1,
- "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
- "items": {
- "type": "object",
- "required": [
- "type",
- "name",
- "value"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
- "minLength": 1
- },
- "name": {
- "type": "string",
- "description": "Human-readable name of the identifier.",
- "minLength": 1
- },
- "url": {
- "type": "string",
- "description": "URL of the identifier's documentation.",
- "format": "uri"
- },
- "value": {
- "type": "string",
- "description": "Value of the identifier, for matching purpose.",
- "minLength": 1
- }
- }
- }
- },
- "links": {
- "type": "array",
- "description": "An array of references to external documentation or articles that describe the vulnerability.",
- "items": {
- "type": "object",
- "required": [
- "url"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the vulnerability details link."
- },
- "url": {
- "type": "string",
- "description": "URL of the vulnerability details document.",
- "format": "uri"
- }
- }
- }
- },
- "details": {
- "$ref": "#/definitions/named_list/properties/items"
- },
- "tracking": {
- "description": "Describes how this vulnerability should be tracked as the project changes.",
- "oneOf": [
- {
- "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
- "required": [
- "items"
- ],
- "properties": {
- "type": {
- "const": "source"
- },
- "items": {
- "type": "array",
- "items": {
- "description": "An item that should be tracked using source-specific tracking methods.",
- "type": "object",
- "required": [
- "signatures"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the file that includes the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the file that includes the vulnerability."
- },
- "signatures": {
- "type": "array",
- "description": "An array of calculated tracking signatures for this tracking item.",
- "minItems": 1,
- "items": {
- "description": "A calculated tracking signature value and metadata.",
- "required": [
- "algorithm",
- "value"
- ],
- "properties": {
- "algorithm": {
- "type": "string",
- "description": "The algorithm used to generate the signature."
- },
- "value": {
- "type": "string",
- "description": "The result of this signature algorithm."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "Each tracking type must declare its own type."
- }
- }
- },
- "flags": {
- "description": "Flags that can be attached to vulnerabilities.",
- "type": "array",
- "items": {
- "type": "object",
- "description": "Informational flags identified and assigned to a vulnerability.",
- "required": [
- "type",
- "origin",
- "description"
- ],
- "properties": {
- "type": {
- "type": "string",
- "minLength": 1,
- "description": "Result of the scan.",
- "enum": [
- "flagged-as-likely-false-positive"
- ]
- },
- "origin": {
- "minLength": 1,
- "description": "Tool that issued the flag.",
- "type": "string"
- },
- "description": {
- "minLength": 1,
- "description": "What the flag is about.",
- "type": "string"
- }
- }
- }
- },
- "location": {
- "required": [
- "commit"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located"
- },
- "commit": {
- "type": "object",
- "description": "Represents the commit in which the vulnerability was detected",
- "required": [
- "sha"
- ],
- "properties": {
- "author": {
- "type": "string"
- },
- "date": {
- "type": "string"
- },
- "message": {
- "type": "string"
- },
- "sha": {
- "type": "string",
- "minLength": 1
- }
- }
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the code affected by the vulnerability"
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the code affected by the vulnerability"
- },
- "class": {
- "type": "string",
- "description": "Provides the name of the class where the vulnerability is located"
- },
- "method": {
- "type": "string",
- "description": "Provides the name of the method where the vulnerability is located"
- }
- }
- },
- "raw_source_code_extract": {
- "type": "string",
- "description": "Provides an unsanitized excerpt of the affected source code."
- }
- }
- }
- },
- "remediations": {
- "type": "array",
- "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
- "items": {
- "type": "object",
- "required": [
- "fixes",
- "summary",
- "diff"
- ],
- "properties": {
- "fixes": {
- "type": "array",
- "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
- "items": {
- "type": "object",
- "required": [
- "cve"
- ],
- "properties": {
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- }
- }
- }
- },
- "summary": {
- "type": "string",
- "minLength": 1,
- "description": "An overview of how the vulnerabilities were fixed."
- },
- "diff": {
- "type": "string",
- "minLength": 1,
- "description": "A base64-encoded remediation code diff, compatible with git apply."
- }
- }
- }
- }
- }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.5/cluster-image-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.5/cluster-image-scanning-report-format.json
deleted file mode 100644
index 882a21e430a..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.5/cluster-image-scanning-report-format.json
+++ /dev/null
@@ -1,977 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Report format for GitLab Cluster Image Scanning",
- "description": "This schema provides the the report format for Cluster Image Scanning (https://docs.gitlab.com/ee/user/application_security/cluster_image_scanning/).",
- "definitions": {
- "detail_type": {
- "oneOf": [
- {
- "$ref": "#/definitions/named_list"
- },
- {
- "$ref": "#/definitions/list"
- },
- {
- "$ref": "#/definitions/table"
- },
- {
- "$ref": "#/definitions/text"
- },
- {
- "$ref": "#/definitions/url"
- },
- {
- "$ref": "#/definitions/code"
- },
- {
- "$ref": "#/definitions/value"
- },
- {
- "$ref": "#/definitions/diff"
- },
- {
- "$ref": "#/definitions/markdown"
- },
- {
- "$ref": "#/definitions/commit"
- },
- {
- "$ref": "#/definitions/file_location"
- },
- {
- "$ref": "#/definitions/module_location"
- }
- ]
- },
- "text_value": {
- "type": "string"
- },
- "named_field": {
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "$ref": "#/definitions/text_value",
- "minLength": 1
- },
- "description": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "named_list": {
- "type": "object",
- "description": "An object with named and typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "named-list"
- },
- "items": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- {
- "$ref": "#/definitions/named_field"
- },
- {
- "$ref": "#/definitions/detail_type"
- }
- ]
- }
- }
- }
- }
- },
- "list": {
- "type": "object",
- "description": "A list of typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- },
- "table": {
- "type": "object",
- "description": "A table of typed fields",
- "required": [
- "type",
- "rows"
- ],
- "properties": {
- "type": {
- "const": "table"
- },
- "header": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- },
- "rows": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- }
- },
- "text": {
- "type": "object",
- "description": "Raw text",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "text"
- },
- "value": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "url": {
- "type": "object",
- "description": "A single URL",
- "required": [
- "type",
- "href"
- ],
- "properties": {
- "type": {
- "const": "url"
- },
- "text": {
- "$ref": "#/definitions/text_value"
- },
- "href": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "http://mysite.com"
- ]
- }
- }
- },
- "code": {
- "type": "object",
- "description": "A codeblock",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "code"
- },
- "value": {
- "type": "string"
- },
- "lang": {
- "type": "string",
- "description": "A programming language"
- }
- }
- },
- "value": {
- "type": "object",
- "description": "A field that can store a range of types of value",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "value"
- },
- "value": {
- "type": [
- "number",
- "string",
- "boolean"
- ]
- }
- }
- },
- "diff": {
- "type": "object",
- "description": "A diff",
- "required": [
- "type",
- "before",
- "after"
- ],
- "properties": {
- "type": {
- "const": "diff"
- },
- "before": {
- "type": "string"
- },
- "after": {
- "type": "string"
- }
- }
- },
- "markdown": {
- "type": "object",
- "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "markdown"
- },
- "value": {
- "$ref": "#/definitions/text_value",
- "examples": [
- "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
- ]
- }
- }
- },
- "commit": {
- "type": "object",
- "description": "A commit/tag/branch within the GitLab project",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "commit"
- },
- "value": {
- "type": "string",
- "description": "The commit SHA",
- "minLength": 1
- }
- }
- },
- "file_location": {
- "type": "object",
- "description": "A location within a file in the project",
- "required": [
- "type",
- "file_name",
- "line_start"
- ],
- "properties": {
- "type": {
- "const": "file-location"
- },
- "file_name": {
- "type": "string",
- "minLength": 1
- },
- "line_start": {
- "type": "integer"
- },
- "line_end": {
- "type": "integer"
- }
- }
- },
- "module_location": {
- "type": "object",
- "description": "A location within a binary module of the form module+relative_offset",
- "required": [
- "type",
- "module_name",
- "offset"
- ],
- "properties": {
- "type": {
- "const": "module-location"
- },
- "module_name": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "compiled_binary"
- ]
- },
- "offset": {
- "type": "integer",
- "examples": [
- 100
- ]
- }
- }
- }
- },
- "self": {
- "version": "14.0.5"
- },
- "required": [
- "version",
- "vulnerabilities"
- ],
- "additionalProperties": true,
- "properties": {
- "scan": {
- "type": "object",
- "required": [
- "end_time",
- "scanner",
- "start_time",
- "status",
- "type"
- ],
- "properties": {
- "end_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-01-28T03:26:02"
- ]
- },
- "messages": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Communication intended for the initiator of a scan.",
- "required": [
- "level",
- "value"
- ],
- "properties": {
- "level": {
- "type": "string",
- "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
- "enum": [
- "info",
- "warn",
- "fatal"
- ],
- "examples": [
- "info"
- ]
- },
- "value": {
- "type": "string",
- "description": "The message to communicate.",
- "minLength": 1,
- "examples": [
- "Permission denied, scanning aborted"
- ]
- }
- }
- }
- },
- "analyzer": {
- "type": "object",
- "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the analyzer.",
- "minLength": 1,
- "examples": [
- "gitlab-dast"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the analyzer, not required to be unique.",
- "minLength": 1,
- "examples": [
- "GitLab DAST"
- ]
- },
- "url": {
- "type": "string",
- "format": "uri",
- "pattern": "^https?://.+",
- "description": "A link to more information about the analyzer.",
- "examples": [
- "https://docs.gitlab.com/ee/user/application_security/dast"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the analyzer.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- },
- "version": {
- "type": "string",
- "description": "The version of the analyzer.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- }
- }
- },
- "scanner": {
- "type": "object",
- "description": "Object defining the scanner used to perform the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the scanner.",
- "minLength": 1,
- "examples": [
- "my-sast-scanner"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the scanner, not required to be unique.",
- "minLength": 1,
- "examples": [
- "My SAST Scanner"
- ]
- },
- "url": {
- "type": "string",
- "description": "A link to more information about the scanner.",
- "examples": [
- "https://scanner.url"
- ]
- },
- "version": {
- "type": "string",
- "description": "The version of the scanner.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the scanner.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- }
- }
- },
- "start_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-02-14T16:01:59"
- ]
- },
- "status": {
- "type": "string",
- "description": "Result of the scan.",
- "enum": [
- "success",
- "failure"
- ]
- },
- "type": {
- "type": "string",
- "description": "Type of the scan.",
- "enum": [
- "cluster_image_scanning"
- ]
- }
- }
- },
- "schema": {
- "type": "string",
- "description": "URI pointing to the validating security report schema.",
- "format": "uri"
- },
- "version": {
- "type": "string",
- "description": "The version of the schema to which the JSON report conforms.",
- "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
- },
- "vulnerabilities": {
- "type": "array",
- "description": "Array of vulnerability objects.",
- "items": {
- "type": "object",
- "description": "Describes the vulnerability.",
- "required": [
- "category",
- "cve",
- "identifiers",
- "location",
- "scanner"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
- "examples": [
- "642735a5-1425-428d-8d4e-3c854885a3c9"
- ]
- },
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
- "name": {
- "type": "string",
- "description": "The name of the vulnerability. This must not include the finding's specific information."
- },
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
- "description": {
- "type": "string",
- "description": "A long text section describing the vulnerability more fully."
- },
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
- "severity": {
- "type": "string",
- "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Info",
- "Unknown",
- "Low",
- "Medium",
- "High",
- "Critical"
- ]
- },
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
- "solution": {
- "type": "string",
- "description": "Explanation of how to fix the vulnerability."
- },
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
- "identifiers": {
- "type": "array",
- "minItems": 1,
- "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
- "items": {
- "type": "object",
- "required": [
- "type",
- "name",
- "value"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
- "minLength": 1
- },
- "name": {
- "type": "string",
- "description": "Human-readable name of the identifier.",
- "minLength": 1
- },
- "url": {
- "type": "string",
- "description": "URL of the identifier's documentation.",
- "format": "uri"
- },
- "value": {
- "type": "string",
- "description": "Value of the identifier, for matching purpose.",
- "minLength": 1
- }
- }
- }
- },
- "links": {
- "type": "array",
- "description": "An array of references to external documentation or articles that describe the vulnerability.",
- "items": {
- "type": "object",
- "required": [
- "url"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the vulnerability details link."
- },
- "url": {
- "type": "string",
- "description": "URL of the vulnerability details document.",
- "format": "uri"
- }
- }
- }
- },
- "details": {
- "$ref": "#/definitions/named_list/properties/items"
- },
- "tracking": {
- "description": "Describes how this vulnerability should be tracked as the project changes.",
- "oneOf": [
- {
- "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
- "required": [
- "items"
- ],
- "properties": {
- "type": {
- "const": "source"
- },
- "items": {
- "type": "array",
- "items": {
- "description": "An item that should be tracked using source-specific tracking methods.",
- "type": "object",
- "required": [
- "signatures"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the file that includes the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the file that includes the vulnerability."
- },
- "signatures": {
- "type": "array",
- "description": "An array of calculated tracking signatures for this tracking item.",
- "minItems": 1,
- "items": {
- "description": "A calculated tracking signature value and metadata.",
- "required": [
- "algorithm",
- "value"
- ],
- "properties": {
- "algorithm": {
- "type": "string",
- "description": "The algorithm used to generate the signature."
- },
- "value": {
- "type": "string",
- "description": "The result of this signature algorithm."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "Each tracking type must declare its own type."
- }
- }
- },
- "flags": {
- "description": "Flags that can be attached to vulnerabilities.",
- "type": "array",
- "items": {
- "type": "object",
- "description": "Informational flags identified and assigned to a vulnerability.",
- "required": [
- "type",
- "origin",
- "description"
- ],
- "properties": {
- "type": {
- "type": "string",
- "minLength": 1,
- "description": "Result of the scan.",
- "enum": [
- "flagged-as-likely-false-positive"
- ]
- },
- "origin": {
- "minLength": 1,
- "description": "Tool that issued the flag.",
- "type": "string"
- },
- "description": {
- "minLength": 1,
- "description": "What the flag is about.",
- "type": "string"
- }
- }
- }
- },
- "location": {
- "type": "object",
- "description": "Identifies the vulnerability's location.",
- "required": [
- "dependency",
- "image",
- "kubernetes_resource"
- ],
- "properties": {
- "dependency": {
- "type": "object",
- "description": "Describes the dependency of a project where the vulnerability is located.",
- "properties": {
- "package": {
- "type": "object",
- "description": "Provides information on the package where the vulnerability is located.",
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the package where the vulnerability is located."
- }
- }
- },
- "version": {
- "type": "string",
- "description": "Version of the vulnerable package."
- },
- "iid": {
- "description": "ID that identifies the dependency in the scope of a dependency file.",
- "type": "number"
- },
- "direct": {
- "type": "boolean",
- "description": "Tells whether this is a direct, top-level dependency of the scanned project."
- },
- "dependency_path": {
- "type": "array",
- "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.",
- "items": {
- "type": "object",
- "required": [
- "iid"
- ],
- "properties": {
- "iid": {
- "type": "number",
- "description": "ID that is unique in the scope of a parent object, and specific to the resource type."
- }
- }
- }
- }
- }
- },
- "operating_system": {
- "type": "string",
- "minLength": 1,
- "maxLength": 255,
- "description": "The operating system that contains the vulnerable package."
- },
- "image": {
- "type": "string",
- "minLength": 1,
- "description": "The analyzed Docker image.",
- "examples": [
- "index.docker.io/library/nginx:1.21"
- ]
- },
- "kubernetes_resource": {
- "type": "object",
- "description": "The specific Kubernetes resource that was scanned.",
- "required": [
- "namespace",
- "kind",
- "name",
- "container_name"
- ],
- "properties": {
- "namespace": {
- "type": "string",
- "minLength": 1,
- "maxLength": 255,
- "description": "The Kubernetes namespace the resource that had its image scanned.",
- "examples": [
- "default",
- "staging",
- "production"
- ]
- },
- "kind": {
- "type": "string",
- "minLength": 1,
- "maxLength": 255,
- "description": "The Kubernetes kind the resource that had its image scanned.",
- "examples": [
- "Deployment",
- "DaemonSet"
- ]
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "maxLength": 255,
- "description": "The name of the resource that had its image scanned.",
- "examples": [
- "nginx-ingress"
- ]
- },
- "container_name": {
- "type": "string",
- "minLength": 1,
- "maxLength": 255,
- "description": "The name of the container that had its image scanned.",
- "examples": [
- "nginx"
- ]
- },
- "agent_id": {
- "type": "string",
- "minLength": 1,
- "maxLength": 255,
- "description": "The GitLab ID of the Kubernetes Agent which performed the scan.",
- "examples": [
- "1234"
- ]
- },
- "cluster_id": {
- "type": "string",
- "minLength": 1,
- "maxLength": 255,
- "description": "The GitLab ID of the Kubernetes cluster when using cluster integration.",
- "examples": [
- "1234"
- ]
- }
- }
- }
- }
- }
- }
- }
- },
- "remediations": {
- "type": "array",
- "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
- "items": {
- "type": "object",
- "required": [
- "fixes",
- "summary",
- "diff"
- ],
- "properties": {
- "fixes": {
- "type": "array",
- "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
- "items": {
- "type": "object",
- "required": [
- "cve"
- ],
- "properties": {
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- }
- }
- }
- },
- "summary": {
- "type": "string",
- "minLength": 1,
- "description": "An overview of how the vulnerabilities were fixed."
- },
- "diff": {
- "type": "string",
- "minLength": 1,
- "description": "A base64-encoded remediation code diff, compatible with git apply."
- }
- }
- }
- }
- }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.5/container-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.5/container-scanning-report-format.json
deleted file mode 100644
index 08f38650340..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.5/container-scanning-report-format.json
+++ /dev/null
@@ -1,910 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Report format for GitLab Container Scanning",
- "description": "This schema provides the the report format for Container Scanning (https://docs.gitlab.com/ee/user/application_security/container_scanning).",
- "definitions": {
- "detail_type": {
- "oneOf": [
- {
- "$ref": "#/definitions/named_list"
- },
- {
- "$ref": "#/definitions/list"
- },
- {
- "$ref": "#/definitions/table"
- },
- {
- "$ref": "#/definitions/text"
- },
- {
- "$ref": "#/definitions/url"
- },
- {
- "$ref": "#/definitions/code"
- },
- {
- "$ref": "#/definitions/value"
- },
- {
- "$ref": "#/definitions/diff"
- },
- {
- "$ref": "#/definitions/markdown"
- },
- {
- "$ref": "#/definitions/commit"
- },
- {
- "$ref": "#/definitions/file_location"
- },
- {
- "$ref": "#/definitions/module_location"
- }
- ]
- },
- "text_value": {
- "type": "string"
- },
- "named_field": {
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "$ref": "#/definitions/text_value",
- "minLength": 1
- },
- "description": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "named_list": {
- "type": "object",
- "description": "An object with named and typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "named-list"
- },
- "items": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- {
- "$ref": "#/definitions/named_field"
- },
- {
- "$ref": "#/definitions/detail_type"
- }
- ]
- }
- }
- }
- }
- },
- "list": {
- "type": "object",
- "description": "A list of typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- },
- "table": {
- "type": "object",
- "description": "A table of typed fields",
- "required": [
- "type",
- "rows"
- ],
- "properties": {
- "type": {
- "const": "table"
- },
- "header": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- },
- "rows": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- }
- },
- "text": {
- "type": "object",
- "description": "Raw text",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "text"
- },
- "value": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "url": {
- "type": "object",
- "description": "A single URL",
- "required": [
- "type",
- "href"
- ],
- "properties": {
- "type": {
- "const": "url"
- },
- "text": {
- "$ref": "#/definitions/text_value"
- },
- "href": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "http://mysite.com"
- ]
- }
- }
- },
- "code": {
- "type": "object",
- "description": "A codeblock",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "code"
- },
- "value": {
- "type": "string"
- },
- "lang": {
- "type": "string",
- "description": "A programming language"
- }
- }
- },
- "value": {
- "type": "object",
- "description": "A field that can store a range of types of value",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "value"
- },
- "value": {
- "type": [
- "number",
- "string",
- "boolean"
- ]
- }
- }
- },
- "diff": {
- "type": "object",
- "description": "A diff",
- "required": [
- "type",
- "before",
- "after"
- ],
- "properties": {
- "type": {
- "const": "diff"
- },
- "before": {
- "type": "string"
- },
- "after": {
- "type": "string"
- }
- }
- },
- "markdown": {
- "type": "object",
- "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "markdown"
- },
- "value": {
- "$ref": "#/definitions/text_value",
- "examples": [
- "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
- ]
- }
- }
- },
- "commit": {
- "type": "object",
- "description": "A commit/tag/branch within the GitLab project",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "commit"
- },
- "value": {
- "type": "string",
- "description": "The commit SHA",
- "minLength": 1
- }
- }
- },
- "file_location": {
- "type": "object",
- "description": "A location within a file in the project",
- "required": [
- "type",
- "file_name",
- "line_start"
- ],
- "properties": {
- "type": {
- "const": "file-location"
- },
- "file_name": {
- "type": "string",
- "minLength": 1
- },
- "line_start": {
- "type": "integer"
- },
- "line_end": {
- "type": "integer"
- }
- }
- },
- "module_location": {
- "type": "object",
- "description": "A location within a binary module of the form module+relative_offset",
- "required": [
- "type",
- "module_name",
- "offset"
- ],
- "properties": {
- "type": {
- "const": "module-location"
- },
- "module_name": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "compiled_binary"
- ]
- },
- "offset": {
- "type": "integer",
- "examples": [
- 100
- ]
- }
- }
- }
- },
- "self": {
- "version": "14.0.5"
- },
- "required": [
- "version",
- "vulnerabilities"
- ],
- "additionalProperties": true,
- "properties": {
- "scan": {
- "type": "object",
- "required": [
- "end_time",
- "scanner",
- "start_time",
- "status",
- "type"
- ],
- "properties": {
- "end_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-01-28T03:26:02"
- ]
- },
- "messages": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Communication intended for the initiator of a scan.",
- "required": [
- "level",
- "value"
- ],
- "properties": {
- "level": {
- "type": "string",
- "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
- "enum": [
- "info",
- "warn",
- "fatal"
- ],
- "examples": [
- "info"
- ]
- },
- "value": {
- "type": "string",
- "description": "The message to communicate.",
- "minLength": 1,
- "examples": [
- "Permission denied, scanning aborted"
- ]
- }
- }
- }
- },
- "analyzer": {
- "type": "object",
- "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the analyzer.",
- "minLength": 1,
- "examples": [
- "gitlab-dast"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the analyzer, not required to be unique.",
- "minLength": 1,
- "examples": [
- "GitLab DAST"
- ]
- },
- "url": {
- "type": "string",
- "format": "uri",
- "pattern": "^https?://.+",
- "description": "A link to more information about the analyzer.",
- "examples": [
- "https://docs.gitlab.com/ee/user/application_security/dast"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the analyzer.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- },
- "version": {
- "type": "string",
- "description": "The version of the analyzer.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- }
- }
- },
- "scanner": {
- "type": "object",
- "description": "Object defining the scanner used to perform the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the scanner.",
- "minLength": 1,
- "examples": [
- "my-sast-scanner"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the scanner, not required to be unique.",
- "minLength": 1,
- "examples": [
- "My SAST Scanner"
- ]
- },
- "url": {
- "type": "string",
- "description": "A link to more information about the scanner.",
- "examples": [
- "https://scanner.url"
- ]
- },
- "version": {
- "type": "string",
- "description": "The version of the scanner.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the scanner.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- }
- }
- },
- "start_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-02-14T16:01:59"
- ]
- },
- "status": {
- "type": "string",
- "description": "Result of the scan.",
- "enum": [
- "success",
- "failure"
- ]
- },
- "type": {
- "type": "string",
- "description": "Type of the scan.",
- "enum": [
- "container_scanning"
- ]
- }
- }
- },
- "schema": {
- "type": "string",
- "description": "URI pointing to the validating security report schema.",
- "format": "uri"
- },
- "version": {
- "type": "string",
- "description": "The version of the schema to which the JSON report conforms.",
- "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
- },
- "vulnerabilities": {
- "type": "array",
- "description": "Array of vulnerability objects.",
- "items": {
- "type": "object",
- "description": "Describes the vulnerability.",
- "required": [
- "category",
- "cve",
- "identifiers",
- "location",
- "scanner"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
- "examples": [
- "642735a5-1425-428d-8d4e-3c854885a3c9"
- ]
- },
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
- "name": {
- "type": "string",
- "description": "The name of the vulnerability. This must not include the finding's specific information."
- },
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
- "description": {
- "type": "string",
- "description": "A long text section describing the vulnerability more fully."
- },
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
- "severity": {
- "type": "string",
- "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Info",
- "Unknown",
- "Low",
- "Medium",
- "High",
- "Critical"
- ]
- },
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
- "solution": {
- "type": "string",
- "description": "Explanation of how to fix the vulnerability."
- },
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
- "identifiers": {
- "type": "array",
- "minItems": 1,
- "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
- "items": {
- "type": "object",
- "required": [
- "type",
- "name",
- "value"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
- "minLength": 1
- },
- "name": {
- "type": "string",
- "description": "Human-readable name of the identifier.",
- "minLength": 1
- },
- "url": {
- "type": "string",
- "description": "URL of the identifier's documentation.",
- "format": "uri"
- },
- "value": {
- "type": "string",
- "description": "Value of the identifier, for matching purpose.",
- "minLength": 1
- }
- }
- }
- },
- "links": {
- "type": "array",
- "description": "An array of references to external documentation or articles that describe the vulnerability.",
- "items": {
- "type": "object",
- "required": [
- "url"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the vulnerability details link."
- },
- "url": {
- "type": "string",
- "description": "URL of the vulnerability details document.",
- "format": "uri"
- }
- }
- }
- },
- "details": {
- "$ref": "#/definitions/named_list/properties/items"
- },
- "tracking": {
- "description": "Describes how this vulnerability should be tracked as the project changes.",
- "oneOf": [
- {
- "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
- "required": [
- "items"
- ],
- "properties": {
- "type": {
- "const": "source"
- },
- "items": {
- "type": "array",
- "items": {
- "description": "An item that should be tracked using source-specific tracking methods.",
- "type": "object",
- "required": [
- "signatures"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the file that includes the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the file that includes the vulnerability."
- },
- "signatures": {
- "type": "array",
- "description": "An array of calculated tracking signatures for this tracking item.",
- "minItems": 1,
- "items": {
- "description": "A calculated tracking signature value and metadata.",
- "required": [
- "algorithm",
- "value"
- ],
- "properties": {
- "algorithm": {
- "type": "string",
- "description": "The algorithm used to generate the signature."
- },
- "value": {
- "type": "string",
- "description": "The result of this signature algorithm."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "Each tracking type must declare its own type."
- }
- }
- },
- "flags": {
- "description": "Flags that can be attached to vulnerabilities.",
- "type": "array",
- "items": {
- "type": "object",
- "description": "Informational flags identified and assigned to a vulnerability.",
- "required": [
- "type",
- "origin",
- "description"
- ],
- "properties": {
- "type": {
- "type": "string",
- "minLength": 1,
- "description": "Result of the scan.",
- "enum": [
- "flagged-as-likely-false-positive"
- ]
- },
- "origin": {
- "minLength": 1,
- "description": "Tool that issued the flag.",
- "type": "string"
- },
- "description": {
- "minLength": 1,
- "description": "What the flag is about.",
- "type": "string"
- }
- }
- }
- },
- "location": {
- "type": "object",
- "description": "Identifies the vulnerability's location.",
- "required": [
- "dependency",
- "operating_system",
- "image"
- ],
- "properties": {
- "dependency": {
- "type": "object",
- "description": "Describes the dependency of a project where the vulnerability is located.",
- "properties": {
- "package": {
- "type": "object",
- "description": "Provides information on the package where the vulnerability is located.",
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the package where the vulnerability is located."
- }
- }
- },
- "version": {
- "type": "string",
- "description": "Version of the vulnerable package."
- },
- "iid": {
- "description": "ID that identifies the dependency in the scope of a dependency file.",
- "type": "number"
- },
- "direct": {
- "type": "boolean",
- "description": "Tells whether this is a direct, top-level dependency of the scanned project."
- },
- "dependency_path": {
- "type": "array",
- "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.",
- "items": {
- "type": "object",
- "required": [
- "iid"
- ],
- "properties": {
- "iid": {
- "type": "number",
- "description": "ID that is unique in the scope of a parent object, and specific to the resource type."
- }
- }
- }
- }
- }
- },
- "operating_system": {
- "type": "string",
- "minLength": 1,
- "description": "The operating system that contains the vulnerable package."
- },
- "image": {
- "type": "string",
- "minLength": 1,
- "description": "The analyzed Docker image."
- },
- "default_branch_image": {
- "type": "string",
- "maxLength": 255,
- "pattern": "^[a-zA-Z0-9/_.-]+:[a-zA-Z0-9_.-]+$",
- "description": "The name of the image on the default branch."
- }
- }
- }
- }
- }
- },
- "remediations": {
- "type": "array",
- "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
- "items": {
- "type": "object",
- "required": [
- "fixes",
- "summary",
- "diff"
- ],
- "properties": {
- "fixes": {
- "type": "array",
- "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
- "items": {
- "type": "object",
- "required": [
- "cve"
- ],
- "properties": {
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- }
- }
- }
- },
- "summary": {
- "type": "string",
- "minLength": 1,
- "description": "An overview of how the vulnerabilities were fixed."
- },
- "diff": {
- "type": "string",
- "minLength": 1,
- "description": "A base64-encoded remediation code diff, compatible with git apply."
- }
- }
- }
- }
- }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.5/coverage-fuzzing-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.5/coverage-fuzzing-report-format.json
deleted file mode 100644
index a442d38c134..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.5/coverage-fuzzing-report-format.json
+++ /dev/null
@@ -1,874 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Report format for GitLab Fuzz Testing",
- "description": "This schema provides the report format for Coverage Guided Fuzz Testing (https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing).",
- "definitions": {
- "detail_type": {
- "oneOf": [
- {
- "$ref": "#/definitions/named_list"
- },
- {
- "$ref": "#/definitions/list"
- },
- {
- "$ref": "#/definitions/table"
- },
- {
- "$ref": "#/definitions/text"
- },
- {
- "$ref": "#/definitions/url"
- },
- {
- "$ref": "#/definitions/code"
- },
- {
- "$ref": "#/definitions/value"
- },
- {
- "$ref": "#/definitions/diff"
- },
- {
- "$ref": "#/definitions/markdown"
- },
- {
- "$ref": "#/definitions/commit"
- },
- {
- "$ref": "#/definitions/file_location"
- },
- {
- "$ref": "#/definitions/module_location"
- }
- ]
- },
- "text_value": {
- "type": "string"
- },
- "named_field": {
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "$ref": "#/definitions/text_value",
- "minLength": 1
- },
- "description": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "named_list": {
- "type": "object",
- "description": "An object with named and typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "named-list"
- },
- "items": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- {
- "$ref": "#/definitions/named_field"
- },
- {
- "$ref": "#/definitions/detail_type"
- }
- ]
- }
- }
- }
- }
- },
- "list": {
- "type": "object",
- "description": "A list of typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- },
- "table": {
- "type": "object",
- "description": "A table of typed fields",
- "required": [
- "type",
- "rows"
- ],
- "properties": {
- "type": {
- "const": "table"
- },
- "header": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- },
- "rows": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- }
- },
- "text": {
- "type": "object",
- "description": "Raw text",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "text"
- },
- "value": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "url": {
- "type": "object",
- "description": "A single URL",
- "required": [
- "type",
- "href"
- ],
- "properties": {
- "type": {
- "const": "url"
- },
- "text": {
- "$ref": "#/definitions/text_value"
- },
- "href": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "http://mysite.com"
- ]
- }
- }
- },
- "code": {
- "type": "object",
- "description": "A codeblock",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "code"
- },
- "value": {
- "type": "string"
- },
- "lang": {
- "type": "string",
- "description": "A programming language"
- }
- }
- },
- "value": {
- "type": "object",
- "description": "A field that can store a range of types of value",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "value"
- },
- "value": {
- "type": [
- "number",
- "string",
- "boolean"
- ]
- }
- }
- },
- "diff": {
- "type": "object",
- "description": "A diff",
- "required": [
- "type",
- "before",
- "after"
- ],
- "properties": {
- "type": {
- "const": "diff"
- },
- "before": {
- "type": "string"
- },
- "after": {
- "type": "string"
- }
- }
- },
- "markdown": {
- "type": "object",
- "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "markdown"
- },
- "value": {
- "$ref": "#/definitions/text_value",
- "examples": [
- "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
- ]
- }
- }
- },
- "commit": {
- "type": "object",
- "description": "A commit/tag/branch within the GitLab project",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "commit"
- },
- "value": {
- "type": "string",
- "description": "The commit SHA",
- "minLength": 1
- }
- }
- },
- "file_location": {
- "type": "object",
- "description": "A location within a file in the project",
- "required": [
- "type",
- "file_name",
- "line_start"
- ],
- "properties": {
- "type": {
- "const": "file-location"
- },
- "file_name": {
- "type": "string",
- "minLength": 1
- },
- "line_start": {
- "type": "integer"
- },
- "line_end": {
- "type": "integer"
- }
- }
- },
- "module_location": {
- "type": "object",
- "description": "A location within a binary module of the form module+relative_offset",
- "required": [
- "type",
- "module_name",
- "offset"
- ],
- "properties": {
- "type": {
- "const": "module-location"
- },
- "module_name": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "compiled_binary"
- ]
- },
- "offset": {
- "type": "integer",
- "examples": [
- 100
- ]
- }
- }
- }
- },
- "self": {
- "version": "14.0.5"
- },
- "required": [
- "version",
- "vulnerabilities"
- ],
- "additionalProperties": true,
- "properties": {
- "scan": {
- "type": "object",
- "required": [
- "end_time",
- "scanner",
- "start_time",
- "status",
- "type"
- ],
- "properties": {
- "end_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-01-28T03:26:02"
- ]
- },
- "messages": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Communication intended for the initiator of a scan.",
- "required": [
- "level",
- "value"
- ],
- "properties": {
- "level": {
- "type": "string",
- "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
- "enum": [
- "info",
- "warn",
- "fatal"
- ],
- "examples": [
- "info"
- ]
- },
- "value": {
- "type": "string",
- "description": "The message to communicate.",
- "minLength": 1,
- "examples": [
- "Permission denied, scanning aborted"
- ]
- }
- }
- }
- },
- "analyzer": {
- "type": "object",
- "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the analyzer.",
- "minLength": 1,
- "examples": [
- "gitlab-dast"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the analyzer, not required to be unique.",
- "minLength": 1,
- "examples": [
- "GitLab DAST"
- ]
- },
- "url": {
- "type": "string",
- "format": "uri",
- "pattern": "^https?://.+",
- "description": "A link to more information about the analyzer.",
- "examples": [
- "https://docs.gitlab.com/ee/user/application_security/dast"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the analyzer.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- },
- "version": {
- "type": "string",
- "description": "The version of the analyzer.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- }
- }
- },
- "scanner": {
- "type": "object",
- "description": "Object defining the scanner used to perform the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the scanner.",
- "minLength": 1,
- "examples": [
- "my-sast-scanner"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the scanner, not required to be unique.",
- "minLength": 1,
- "examples": [
- "My SAST Scanner"
- ]
- },
- "url": {
- "type": "string",
- "description": "A link to more information about the scanner.",
- "examples": [
- "https://scanner.url"
- ]
- },
- "version": {
- "type": "string",
- "description": "The version of the scanner.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the scanner.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- }
- }
- },
- "start_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-02-14T16:01:59"
- ]
- },
- "status": {
- "type": "string",
- "description": "Result of the scan.",
- "enum": [
- "success",
- "failure"
- ]
- },
- "type": {
- "type": "string",
- "description": "Type of the scan.",
- "enum": [
- "coverage_fuzzing"
- ]
- }
- }
- },
- "schema": {
- "type": "string",
- "description": "URI pointing to the validating security report schema.",
- "format": "uri"
- },
- "version": {
- "type": "string",
- "description": "The version of the schema to which the JSON report conforms.",
- "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
- },
- "vulnerabilities": {
- "type": "array",
- "description": "Array of vulnerability objects.",
- "items": {
- "type": "object",
- "description": "Describes the vulnerability.",
- "required": [
- "category",
- "cve",
- "identifiers",
- "location",
- "scanner"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
- "examples": [
- "642735a5-1425-428d-8d4e-3c854885a3c9"
- ]
- },
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
- "name": {
- "type": "string",
- "description": "The name of the vulnerability. This must not include the finding's specific information."
- },
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
- "description": {
- "type": "string",
- "description": "A long text section describing the vulnerability more fully."
- },
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
- "severity": {
- "type": "string",
- "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Info",
- "Unknown",
- "Low",
- "Medium",
- "High",
- "Critical"
- ]
- },
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
- "solution": {
- "type": "string",
- "description": "Explanation of how to fix the vulnerability."
- },
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
- "identifiers": {
- "type": "array",
- "minItems": 1,
- "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
- "items": {
- "type": "object",
- "required": [
- "type",
- "name",
- "value"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
- "minLength": 1
- },
- "name": {
- "type": "string",
- "description": "Human-readable name of the identifier.",
- "minLength": 1
- },
- "url": {
- "type": "string",
- "description": "URL of the identifier's documentation.",
- "format": "uri"
- },
- "value": {
- "type": "string",
- "description": "Value of the identifier, for matching purpose.",
- "minLength": 1
- }
- }
- }
- },
- "links": {
- "type": "array",
- "description": "An array of references to external documentation or articles that describe the vulnerability.",
- "items": {
- "type": "object",
- "required": [
- "url"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the vulnerability details link."
- },
- "url": {
- "type": "string",
- "description": "URL of the vulnerability details document.",
- "format": "uri"
- }
- }
- }
- },
- "details": {
- "$ref": "#/definitions/named_list/properties/items"
- },
- "tracking": {
- "description": "Describes how this vulnerability should be tracked as the project changes.",
- "oneOf": [
- {
- "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
- "required": [
- "items"
- ],
- "properties": {
- "type": {
- "const": "source"
- },
- "items": {
- "type": "array",
- "items": {
- "description": "An item that should be tracked using source-specific tracking methods.",
- "type": "object",
- "required": [
- "signatures"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the file that includes the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the file that includes the vulnerability."
- },
- "signatures": {
- "type": "array",
- "description": "An array of calculated tracking signatures for this tracking item.",
- "minItems": 1,
- "items": {
- "description": "A calculated tracking signature value and metadata.",
- "required": [
- "algorithm",
- "value"
- ],
- "properties": {
- "algorithm": {
- "type": "string",
- "description": "The algorithm used to generate the signature."
- },
- "value": {
- "type": "string",
- "description": "The result of this signature algorithm."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "Each tracking type must declare its own type."
- }
- }
- },
- "flags": {
- "description": "Flags that can be attached to vulnerabilities.",
- "type": "array",
- "items": {
- "type": "object",
- "description": "Informational flags identified and assigned to a vulnerability.",
- "required": [
- "type",
- "origin",
- "description"
- ],
- "properties": {
- "type": {
- "type": "string",
- "minLength": 1,
- "description": "Result of the scan.",
- "enum": [
- "flagged-as-likely-false-positive"
- ]
- },
- "origin": {
- "minLength": 1,
- "description": "Tool that issued the flag.",
- "type": "string"
- },
- "description": {
- "minLength": 1,
- "description": "What the flag is about.",
- "type": "string"
- }
- }
- }
- },
- "location": {
- "description": "The location of the error",
- "type": "object",
- "properties": {
- "crash_address": {
- "type": "string",
- "description": "The relative address in memory were the crash occurred.",
- "examples": [
- "0xabababab"
- ]
- },
- "stacktrace_snippet": {
- "type": "string",
- "description": "The stack trace recorded during fuzzing resulting the crash.",
- "examples": [
- "func_a+0xabcd\nfunc_b+0xabcc"
- ]
- },
- "crash_state": {
- "type": "string",
- "description": "Minimised and normalized crash stack-trace (called crash_state).",
- "examples": [
- "func_a+0xa\nfunc_b+0xb\nfunc_c+0xc"
- ]
- },
- "crash_type": {
- "type": "string",
- "description": "Type of the crash.",
- "examples": [
- "Heap-Buffer-overflow",
- "Division-by-zero"
- ]
- }
- }
- }
- }
- }
- },
- "remediations": {
- "type": "array",
- "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
- "items": {
- "type": "object",
- "required": [
- "fixes",
- "summary",
- "diff"
- ],
- "properties": {
- "fixes": {
- "type": "array",
- "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
- "items": {
- "type": "object",
- "required": [
- "cve"
- ],
- "properties": {
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- }
- }
- }
- },
- "summary": {
- "type": "string",
- "minLength": 1,
- "description": "An overview of how the vulnerabilities were fixed."
- },
- "diff": {
- "type": "string",
- "minLength": 1,
- "description": "A base64-encoded remediation code diff, compatible with git apply."
- }
- }
- }
- }
- }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.5/dast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.5/dast-report-format.json
deleted file mode 100644
index 9a4d1515bc2..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.5/dast-report-format.json
+++ /dev/null
@@ -1,1291 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Report format for GitLab DAST",
- "description": "This schema provides the the report format for Dynamic Application Security Testing (https://docs.gitlab.com/ee/user/application_security/dast).",
- "definitions": {
- "detail_type": {
- "oneOf": [
- {
- "$ref": "#/definitions/named_list"
- },
- {
- "$ref": "#/definitions/list"
- },
- {
- "$ref": "#/definitions/table"
- },
- {
- "$ref": "#/definitions/text"
- },
- {
- "$ref": "#/definitions/url"
- },
- {
- "$ref": "#/definitions/code"
- },
- {
- "$ref": "#/definitions/value"
- },
- {
- "$ref": "#/definitions/diff"
- },
- {
- "$ref": "#/definitions/markdown"
- },
- {
- "$ref": "#/definitions/commit"
- },
- {
- "$ref": "#/definitions/file_location"
- },
- {
- "$ref": "#/definitions/module_location"
- }
- ]
- },
- "text_value": {
- "type": "string"
- },
- "named_field": {
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "$ref": "#/definitions/text_value",
- "minLength": 1
- },
- "description": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "named_list": {
- "type": "object",
- "description": "An object with named and typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "named-list"
- },
- "items": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- {
- "$ref": "#/definitions/named_field"
- },
- {
- "$ref": "#/definitions/detail_type"
- }
- ]
- }
- }
- }
- }
- },
- "list": {
- "type": "object",
- "description": "A list of typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- },
- "table": {
- "type": "object",
- "description": "A table of typed fields",
- "required": [
- "type",
- "rows"
- ],
- "properties": {
- "type": {
- "const": "table"
- },
- "header": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- },
- "rows": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- }
- },
- "text": {
- "type": "object",
- "description": "Raw text",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "text"
- },
- "value": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "url": {
- "type": "object",
- "description": "A single URL",
- "required": [
- "type",
- "href"
- ],
- "properties": {
- "type": {
- "const": "url"
- },
- "text": {
- "$ref": "#/definitions/text_value"
- },
- "href": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "http://mysite.com"
- ]
- }
- }
- },
- "code": {
- "type": "object",
- "description": "A codeblock",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "code"
- },
- "value": {
- "type": "string"
- },
- "lang": {
- "type": "string",
- "description": "A programming language"
- }
- }
- },
- "value": {
- "type": "object",
- "description": "A field that can store a range of types of value",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "value"
- },
- "value": {
- "type": [
- "number",
- "string",
- "boolean"
- ]
- }
- }
- },
- "diff": {
- "type": "object",
- "description": "A diff",
- "required": [
- "type",
- "before",
- "after"
- ],
- "properties": {
- "type": {
- "const": "diff"
- },
- "before": {
- "type": "string"
- },
- "after": {
- "type": "string"
- }
- }
- },
- "markdown": {
- "type": "object",
- "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "markdown"
- },
- "value": {
- "$ref": "#/definitions/text_value",
- "examples": [
- "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
- ]
- }
- }
- },
- "commit": {
- "type": "object",
- "description": "A commit/tag/branch within the GitLab project",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "commit"
- },
- "value": {
- "type": "string",
- "description": "The commit SHA",
- "minLength": 1
- }
- }
- },
- "file_location": {
- "type": "object",
- "description": "A location within a file in the project",
- "required": [
- "type",
- "file_name",
- "line_start"
- ],
- "properties": {
- "type": {
- "const": "file-location"
- },
- "file_name": {
- "type": "string",
- "minLength": 1
- },
- "line_start": {
- "type": "integer"
- },
- "line_end": {
- "type": "integer"
- }
- }
- },
- "module_location": {
- "type": "object",
- "description": "A location within a binary module of the form module+relative_offset",
- "required": [
- "type",
- "module_name",
- "offset"
- ],
- "properties": {
- "type": {
- "const": "module-location"
- },
- "module_name": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "compiled_binary"
- ]
- },
- "offset": {
- "type": "integer",
- "examples": [
- 100
- ]
- }
- }
- }
- },
- "self": {
- "version": "14.0.5"
- },
- "required": [
- "version",
- "vulnerabilities"
- ],
- "additionalProperties": true,
- "properties": {
- "scan": {
- "type": "object",
- "required": [
- "end_time",
- "scanned_resources",
- "scanner",
- "start_time",
- "status",
- "type"
- ],
- "properties": {
- "end_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-01-28T03:26:02"
- ]
- },
- "messages": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Communication intended for the initiator of a scan.",
- "required": [
- "level",
- "value"
- ],
- "properties": {
- "level": {
- "type": "string",
- "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
- "enum": [
- "info",
- "warn",
- "fatal"
- ],
- "examples": [
- "info"
- ]
- },
- "value": {
- "type": "string",
- "description": "The message to communicate.",
- "minLength": 1,
- "examples": [
- "Permission denied, scanning aborted"
- ]
- }
- }
- }
- },
- "analyzer": {
- "type": "object",
- "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the analyzer.",
- "minLength": 1,
- "examples": [
- "gitlab-dast"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the analyzer, not required to be unique.",
- "minLength": 1,
- "examples": [
- "GitLab DAST"
- ]
- },
- "url": {
- "type": "string",
- "format": "uri",
- "pattern": "^https?://.+",
- "description": "A link to more information about the analyzer.",
- "examples": [
- "https://docs.gitlab.com/ee/user/application_security/dast"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the analyzer.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- },
- "version": {
- "type": "string",
- "description": "The version of the analyzer.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- }
- }
- },
- "scanner": {
- "type": "object",
- "description": "Object defining the scanner used to perform the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the scanner.",
- "minLength": 1,
- "examples": [
- "my-sast-scanner"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the scanner, not required to be unique.",
- "minLength": 1,
- "examples": [
- "My SAST Scanner"
- ]
- },
- "url": {
- "type": "string",
- "description": "A link to more information about the scanner.",
- "examples": [
- "https://scanner.url"
- ]
- },
- "version": {
- "type": "string",
- "description": "The version of the scanner.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the scanner.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- }
- }
- },
- "start_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-02-14T16:01:59"
- ]
- },
- "status": {
- "type": "string",
- "description": "Result of the scan.",
- "enum": [
- "success",
- "failure"
- ]
- },
- "type": {
- "type": "string",
- "description": "Type of the scan.",
- "enum": [
- "dast",
- "api_fuzzing"
- ]
- },
- "scanned_resources": {
- "type": "array",
- "description": "The attack surface scanned by DAST.",
- "items": {
- "type": "object",
- "required": [
- "method",
- "url",
- "type"
- ],
- "properties": {
- "method": {
- "type": "string",
- "minLength": 1,
- "description": "HTTP method of the scanned resource.",
- "examples": [
- "GET",
- "POST",
- "HEAD"
- ]
- },
- "url": {
- "type": "string",
- "minLength": 1,
- "description": "URL of the scanned resource.",
- "examples": [
- "http://my.site.com/a-page"
- ]
- },
- "type": {
- "type": "string",
- "minLength": 1,
- "description": "Type of the scanned resource, for DAST, this must be 'url'.",
- "examples": [
- "url"
- ]
- }
- }
- }
- }
- }
- },
- "schema": {
- "type": "string",
- "description": "URI pointing to the validating security report schema.",
- "format": "uri"
- },
- "version": {
- "type": "string",
- "description": "The version of the schema to which the JSON report conforms.",
- "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
- },
- "vulnerabilities": {
- "type": "array",
- "description": "Array of vulnerability objects.",
- "items": {
- "type": "object",
- "description": "Describes the vulnerability.",
- "required": [
- "category",
- "cve",
- "identifiers",
- "location",
- "scanner"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
- "examples": [
- "642735a5-1425-428d-8d4e-3c854885a3c9"
- ]
- },
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
- "name": {
- "type": "string",
- "description": "The name of the vulnerability. This must not include the finding's specific information."
- },
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
- "description": {
- "type": "string",
- "description": "A long text section describing the vulnerability more fully."
- },
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
- "severity": {
- "type": "string",
- "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Info",
- "Unknown",
- "Low",
- "Medium",
- "High",
- "Critical"
- ]
- },
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
- "solution": {
- "type": "string",
- "description": "Explanation of how to fix the vulnerability."
- },
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
- "identifiers": {
- "type": "array",
- "minItems": 1,
- "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
- "items": {
- "type": "object",
- "required": [
- "type",
- "name",
- "value"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
- "minLength": 1
- },
- "name": {
- "type": "string",
- "description": "Human-readable name of the identifier.",
- "minLength": 1
- },
- "url": {
- "type": "string",
- "description": "URL of the identifier's documentation.",
- "format": "uri"
- },
- "value": {
- "type": "string",
- "description": "Value of the identifier, for matching purpose.",
- "minLength": 1
- }
- }
- }
- },
- "links": {
- "type": "array",
- "description": "An array of references to external documentation or articles that describe the vulnerability.",
- "items": {
- "type": "object",
- "required": [
- "url"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the vulnerability details link."
- },
- "url": {
- "type": "string",
- "description": "URL of the vulnerability details document.",
- "format": "uri"
- }
- }
- }
- },
- "details": {
- "$ref": "#/definitions/named_list/properties/items"
- },
- "tracking": {
- "description": "Describes how this vulnerability should be tracked as the project changes.",
- "oneOf": [
- {
- "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
- "required": [
- "items"
- ],
- "properties": {
- "type": {
- "const": "source"
- },
- "items": {
- "type": "array",
- "items": {
- "description": "An item that should be tracked using source-specific tracking methods.",
- "type": "object",
- "required": [
- "signatures"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the file that includes the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the file that includes the vulnerability."
- },
- "signatures": {
- "type": "array",
- "description": "An array of calculated tracking signatures for this tracking item.",
- "minItems": 1,
- "items": {
- "description": "A calculated tracking signature value and metadata.",
- "required": [
- "algorithm",
- "value"
- ],
- "properties": {
- "algorithm": {
- "type": "string",
- "description": "The algorithm used to generate the signature."
- },
- "value": {
- "type": "string",
- "description": "The result of this signature algorithm."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "Each tracking type must declare its own type."
- }
- }
- },
- "flags": {
- "description": "Flags that can be attached to vulnerabilities.",
- "type": "array",
- "items": {
- "type": "object",
- "description": "Informational flags identified and assigned to a vulnerability.",
- "required": [
- "type",
- "origin",
- "description"
- ],
- "properties": {
- "type": {
- "type": "string",
- "minLength": 1,
- "description": "Result of the scan.",
- "enum": [
- "flagged-as-likely-false-positive"
- ]
- },
- "origin": {
- "minLength": 1,
- "description": "Tool that issued the flag.",
- "type": "string"
- },
- "description": {
- "minLength": 1,
- "description": "What the flag is about.",
- "type": "string"
- }
- }
- }
- },
- "evidence": {
- "type": "object",
- "properties": {
- "source": {
- "type": "object",
- "description": "Source of evidence",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "Unique source identifier",
- "examples": [
- "assert:LogAnalysis",
- "assert:StatusCode"
- ]
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Source display name",
- "examples": [
- "Log Analysis",
- "Status Code"
- ]
- },
- "url": {
- "type": "string",
- "description": "Link to additional information",
- "examples": [
- "https://docs.gitlab.com/ee/development/integrations/secure.html"
- ]
- }
- }
- },
- "summary": {
- "type": "string",
- "description": "Human readable string containing evidence of the vulnerability.",
- "examples": [
- "Credit card 4111111111111111 found",
- "Server leaked information nginx/1.17.6"
- ]
- },
- "request": {
- "type": "object",
- "description": "An HTTP request.",
- "required": [
- "headers",
- "method",
- "url"
- ],
- "properties": {
- "headers": {
- "type": "array",
- "description": "HTTP headers present on the request.",
- "items": {
- "type": "object",
- "required": [
- "name",
- "value"
- ],
- "properties": {
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Name of the HTTP header.",
- "examples": [
- "Accept",
- "Content-Length",
- "Content-Type"
- ]
- },
- "value": {
- "type": "string",
- "minLength": 1,
- "description": "Value of the HTTP header.",
- "examples": [
- "*/*",
- "560",
- "application/json; charset=utf-8"
- ]
- }
- }
- }
- },
- "method": {
- "type": "string",
- "minLength": 1,
- "description": "HTTP method used in the request.",
- "examples": [
- "GET",
- "POST"
- ]
- },
- "url": {
- "type": "string",
- "minLength": 1,
- "description": "URL of the request.",
- "examples": [
- "http://my.site.com/vulnerable-endpoint?show-credit-card"
- ]
- },
- "body": {
- "type": "string",
- "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.",
- "examples": [
- "user=jsmith&first=%27&last=smith"
- ]
- }
- }
- },
- "response": {
- "type": "object",
- "description": "An HTTP response.",
- "required": [
- "headers",
- "reason_phrase",
- "status_code"
- ],
- "properties": {
- "headers": {
- "type": "array",
- "description": "HTTP headers present on the request.",
- "items": {
- "type": "object",
- "required": [
- "name",
- "value"
- ],
- "properties": {
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Name of the HTTP header.",
- "examples": [
- "Accept",
- "Content-Length",
- "Content-Type"
- ]
- },
- "value": {
- "type": "string",
- "minLength": 1,
- "description": "Value of the HTTP header.",
- "examples": [
- "*/*",
- "560",
- "application/json; charset=utf-8"
- ]
- }
- }
- }
- },
- "reason_phrase": {
- "type": "string",
- "description": "HTTP reason phrase of the response.",
- "examples": [
- "OK",
- "Internal Server Error"
- ]
- },
- "status_code": {
- "type": "integer",
- "description": "HTTP status code of the response.",
- "examples": [
- 200,
- 500
- ]
- },
- "body": {
- "type": "string",
- "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.",
- "examples": [
- "{\"user_id\": 2}"
- ]
- }
- }
- },
- "supporting_messages": {
- "type": "array",
- "description": "Array of supporting http messages.",
- "items": {
- "type": "object",
- "description": "A supporting http message.",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Message display name.",
- "examples": [
- "Unmodified",
- "Recorded"
- ]
- },
- "request": {
- "type": "object",
- "description": "An HTTP request.",
- "required": [
- "headers",
- "method",
- "url"
- ],
- "properties": {
- "headers": {
- "type": "array",
- "description": "HTTP headers present on the request.",
- "items": {
- "type": "object",
- "required": [
- "name",
- "value"
- ],
- "properties": {
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Name of the HTTP header.",
- "examples": [
- "Accept",
- "Content-Length",
- "Content-Type"
- ]
- },
- "value": {
- "type": "string",
- "minLength": 1,
- "description": "Value of the HTTP header.",
- "examples": [
- "*/*",
- "560",
- "application/json; charset=utf-8"
- ]
- }
- }
- }
- },
- "method": {
- "type": "string",
- "minLength": 1,
- "description": "HTTP method used in the request.",
- "examples": [
- "GET",
- "POST"
- ]
- },
- "url": {
- "type": "string",
- "minLength": 1,
- "description": "URL of the request.",
- "examples": [
- "http://my.site.com/vulnerable-endpoint?show-credit-card"
- ]
- },
- "body": {
- "type": "string",
- "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.",
- "examples": [
- "user=jsmith&first=%27&last=smith"
- ]
- }
- }
- },
- "response": {
- "type": "object",
- "description": "An HTTP response.",
- "required": [
- "headers",
- "reason_phrase",
- "status_code"
- ],
- "properties": {
- "headers": {
- "type": "array",
- "description": "HTTP headers present on the request.",
- "items": {
- "type": "object",
- "required": [
- "name",
- "value"
- ],
- "properties": {
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Name of the HTTP header.",
- "examples": [
- "Accept",
- "Content-Length",
- "Content-Type"
- ]
- },
- "value": {
- "type": "string",
- "minLength": 1,
- "description": "Value of the HTTP header.",
- "examples": [
- "*/*",
- "560",
- "application/json; charset=utf-8"
- ]
- }
- }
- }
- },
- "reason_phrase": {
- "type": "string",
- "description": "HTTP reason phrase of the response.",
- "examples": [
- "OK",
- "Internal Server Error"
- ]
- },
- "status_code": {
- "type": "integer",
- "description": "HTTP status code of the response.",
- "examples": [
- 200,
- 500
- ]
- },
- "body": {
- "type": "string",
- "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.",
- "examples": [
- "{\"user_id\": 2}"
- ]
- }
- }
- }
- }
- }
- }
- }
- },
- "location": {
- "type": "object",
- "description": "Identifies the vulnerability's location.",
- "properties": {
- "hostname": {
- "type": "string",
- "description": "The protocol, domain, and port of the application where the vulnerability was found."
- },
- "method": {
- "type": "string",
- "description": "The HTTP method that was used to request the URL where the vulnerability was found."
- },
- "param": {
- "type": "string",
- "description": "A value provided by a vulnerability rule related to the found vulnerability. Examples include a header value, or a parameter used in a HTTP POST."
- },
- "path": {
- "type": "string",
- "description": "The path of the URL where the vulnerability was found. Typically, this would start with a forward slash."
- }
- }
- },
- "assets": {
- "type": "array",
- "description": "Array of build assets associated with vulnerability.",
- "items": {
- "type": "object",
- "description": "Describes an asset associated with vulnerability.",
- "required": [
- "type",
- "name",
- "url"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "The type of asset",
- "enum": [
- "http_session",
- "postman"
- ]
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Display name for asset",
- "examples": [
- "HTTP Messages",
- "Postman Collection"
- ]
- },
- "url": {
- "type": "string",
- "minLength": 1,
- "description": "Link to asset in build artifacts",
- "examples": [
- "https://gitlab.com/gitlab-org/security-products/dast/-/jobs/626397001/artifacts/file//output/zap_session.data"
- ]
- }
- }
- }
- },
- "discovered_at": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss.sss, representing when the vulnerability was discovered",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}\\.\\d{3}$",
- "examples": [
- "2020-01-28T03:26:02.956"
- ]
- }
- }
- }
- },
- "remediations": {
- "type": "array",
- "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
- "items": {
- "type": "object",
- "required": [
- "fixes",
- "summary",
- "diff"
- ],
- "properties": {
- "fixes": {
- "type": "array",
- "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
- "items": {
- "type": "object",
- "required": [
- "cve"
- ],
- "properties": {
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- }
- }
- }
- },
- "summary": {
- "type": "string",
- "minLength": 1,
- "description": "An overview of how the vulnerabilities were fixed."
- },
- "diff": {
- "type": "string",
- "minLength": 1,
- "description": "A base64-encoded remediation code diff, compatible with git apply."
- }
- }
- }
- }
- }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.5/dependency-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.5/dependency-scanning-report-format.json
deleted file mode 100644
index e84dd9c87d8..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.5/dependency-scanning-report-format.json
+++ /dev/null
@@ -1,968 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Report format for GitLab Dependency Scanning",
- "description": "This schema provides the the report format for Dependency Scanning analyzers (https://docs.gitlab.com/ee/user/application_security/dependency_scanning).",
- "definitions": {
- "detail_type": {
- "oneOf": [
- {
- "$ref": "#/definitions/named_list"
- },
- {
- "$ref": "#/definitions/list"
- },
- {
- "$ref": "#/definitions/table"
- },
- {
- "$ref": "#/definitions/text"
- },
- {
- "$ref": "#/definitions/url"
- },
- {
- "$ref": "#/definitions/code"
- },
- {
- "$ref": "#/definitions/value"
- },
- {
- "$ref": "#/definitions/diff"
- },
- {
- "$ref": "#/definitions/markdown"
- },
- {
- "$ref": "#/definitions/commit"
- },
- {
- "$ref": "#/definitions/file_location"
- },
- {
- "$ref": "#/definitions/module_location"
- }
- ]
- },
- "text_value": {
- "type": "string"
- },
- "named_field": {
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "$ref": "#/definitions/text_value",
- "minLength": 1
- },
- "description": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "named_list": {
- "type": "object",
- "description": "An object with named and typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "named-list"
- },
- "items": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- {
- "$ref": "#/definitions/named_field"
- },
- {
- "$ref": "#/definitions/detail_type"
- }
- ]
- }
- }
- }
- }
- },
- "list": {
- "type": "object",
- "description": "A list of typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- },
- "table": {
- "type": "object",
- "description": "A table of typed fields",
- "required": [
- "type",
- "rows"
- ],
- "properties": {
- "type": {
- "const": "table"
- },
- "header": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- },
- "rows": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- }
- },
- "text": {
- "type": "object",
- "description": "Raw text",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "text"
- },
- "value": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "url": {
- "type": "object",
- "description": "A single URL",
- "required": [
- "type",
- "href"
- ],
- "properties": {
- "type": {
- "const": "url"
- },
- "text": {
- "$ref": "#/definitions/text_value"
- },
- "href": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "http://mysite.com"
- ]
- }
- }
- },
- "code": {
- "type": "object",
- "description": "A codeblock",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "code"
- },
- "value": {
- "type": "string"
- },
- "lang": {
- "type": "string",
- "description": "A programming language"
- }
- }
- },
- "value": {
- "type": "object",
- "description": "A field that can store a range of types of value",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "value"
- },
- "value": {
- "type": [
- "number",
- "string",
- "boolean"
- ]
- }
- }
- },
- "diff": {
- "type": "object",
- "description": "A diff",
- "required": [
- "type",
- "before",
- "after"
- ],
- "properties": {
- "type": {
- "const": "diff"
- },
- "before": {
- "type": "string"
- },
- "after": {
- "type": "string"
- }
- }
- },
- "markdown": {
- "type": "object",
- "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "markdown"
- },
- "value": {
- "$ref": "#/definitions/text_value",
- "examples": [
- "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
- ]
- }
- }
- },
- "commit": {
- "type": "object",
- "description": "A commit/tag/branch within the GitLab project",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "commit"
- },
- "value": {
- "type": "string",
- "description": "The commit SHA",
- "minLength": 1
- }
- }
- },
- "file_location": {
- "type": "object",
- "description": "A location within a file in the project",
- "required": [
- "type",
- "file_name",
- "line_start"
- ],
- "properties": {
- "type": {
- "const": "file-location"
- },
- "file_name": {
- "type": "string",
- "minLength": 1
- },
- "line_start": {
- "type": "integer"
- },
- "line_end": {
- "type": "integer"
- }
- }
- },
- "module_location": {
- "type": "object",
- "description": "A location within a binary module of the form module+relative_offset",
- "required": [
- "type",
- "module_name",
- "offset"
- ],
- "properties": {
- "type": {
- "const": "module-location"
- },
- "module_name": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "compiled_binary"
- ]
- },
- "offset": {
- "type": "integer",
- "examples": [
- 100
- ]
- }
- }
- }
- },
- "self": {
- "version": "14.0.5"
- },
- "required": [
- "dependency_files",
- "version",
- "vulnerabilities"
- ],
- "additionalProperties": true,
- "properties": {
- "scan": {
- "type": "object",
- "required": [
- "end_time",
- "scanner",
- "start_time",
- "status",
- "type"
- ],
- "properties": {
- "end_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-01-28T03:26:02"
- ]
- },
- "messages": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Communication intended for the initiator of a scan.",
- "required": [
- "level",
- "value"
- ],
- "properties": {
- "level": {
- "type": "string",
- "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
- "enum": [
- "info",
- "warn",
- "fatal"
- ],
- "examples": [
- "info"
- ]
- },
- "value": {
- "type": "string",
- "description": "The message to communicate.",
- "minLength": 1,
- "examples": [
- "Permission denied, scanning aborted"
- ]
- }
- }
- }
- },
- "analyzer": {
- "type": "object",
- "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the analyzer.",
- "minLength": 1,
- "examples": [
- "gitlab-dast"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the analyzer, not required to be unique.",
- "minLength": 1,
- "examples": [
- "GitLab DAST"
- ]
- },
- "url": {
- "type": "string",
- "format": "uri",
- "pattern": "^https?://.+",
- "description": "A link to more information about the analyzer.",
- "examples": [
- "https://docs.gitlab.com/ee/user/application_security/dast"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the analyzer.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- },
- "version": {
- "type": "string",
- "description": "The version of the analyzer.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- }
- }
- },
- "scanner": {
- "type": "object",
- "description": "Object defining the scanner used to perform the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the scanner.",
- "minLength": 1,
- "examples": [
- "my-sast-scanner"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the scanner, not required to be unique.",
- "minLength": 1,
- "examples": [
- "My SAST Scanner"
- ]
- },
- "url": {
- "type": "string",
- "description": "A link to more information about the scanner.",
- "examples": [
- "https://scanner.url"
- ]
- },
- "version": {
- "type": "string",
- "description": "The version of the scanner.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the scanner.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- }
- }
- },
- "start_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-02-14T16:01:59"
- ]
- },
- "status": {
- "type": "string",
- "description": "Result of the scan.",
- "enum": [
- "success",
- "failure"
- ]
- },
- "type": {
- "type": "string",
- "description": "Type of the scan.",
- "enum": [
- "dependency_scanning"
- ]
- }
- }
- },
- "schema": {
- "type": "string",
- "description": "URI pointing to the validating security report schema.",
- "format": "uri"
- },
- "version": {
- "type": "string",
- "description": "The version of the schema to which the JSON report conforms.",
- "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
- },
- "vulnerabilities": {
- "type": "array",
- "description": "Array of vulnerability objects.",
- "items": {
- "type": "object",
- "description": "Describes the vulnerability.",
- "required": [
- "category",
- "cve",
- "identifiers",
- "location",
- "scanner"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
- "examples": [
- "642735a5-1425-428d-8d4e-3c854885a3c9"
- ]
- },
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
- "name": {
- "type": "string",
- "description": "The name of the vulnerability. This must not include the finding's specific information."
- },
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
- "description": {
- "type": "string",
- "description": "A long text section describing the vulnerability more fully."
- },
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
- "severity": {
- "type": "string",
- "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Info",
- "Unknown",
- "Low",
- "Medium",
- "High",
- "Critical"
- ]
- },
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
- "solution": {
- "type": "string",
- "description": "Explanation of how to fix the vulnerability."
- },
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
- "identifiers": {
- "type": "array",
- "minItems": 1,
- "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
- "items": {
- "type": "object",
- "required": [
- "type",
- "name",
- "value"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
- "minLength": 1
- },
- "name": {
- "type": "string",
- "description": "Human-readable name of the identifier.",
- "minLength": 1
- },
- "url": {
- "type": "string",
- "description": "URL of the identifier's documentation.",
- "format": "uri"
- },
- "value": {
- "type": "string",
- "description": "Value of the identifier, for matching purpose.",
- "minLength": 1
- }
- }
- }
- },
- "links": {
- "type": "array",
- "description": "An array of references to external documentation or articles that describe the vulnerability.",
- "items": {
- "type": "object",
- "required": [
- "url"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the vulnerability details link."
- },
- "url": {
- "type": "string",
- "description": "URL of the vulnerability details document.",
- "format": "uri"
- }
- }
- }
- },
- "details": {
- "$ref": "#/definitions/named_list/properties/items"
- },
- "tracking": {
- "description": "Describes how this vulnerability should be tracked as the project changes.",
- "oneOf": [
- {
- "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
- "required": [
- "items"
- ],
- "properties": {
- "type": {
- "const": "source"
- },
- "items": {
- "type": "array",
- "items": {
- "description": "An item that should be tracked using source-specific tracking methods.",
- "type": "object",
- "required": [
- "signatures"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the file that includes the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the file that includes the vulnerability."
- },
- "signatures": {
- "type": "array",
- "description": "An array of calculated tracking signatures for this tracking item.",
- "minItems": 1,
- "items": {
- "description": "A calculated tracking signature value and metadata.",
- "required": [
- "algorithm",
- "value"
- ],
- "properties": {
- "algorithm": {
- "type": "string",
- "description": "The algorithm used to generate the signature."
- },
- "value": {
- "type": "string",
- "description": "The result of this signature algorithm."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "Each tracking type must declare its own type."
- }
- }
- },
- "flags": {
- "description": "Flags that can be attached to vulnerabilities.",
- "type": "array",
- "items": {
- "type": "object",
- "description": "Informational flags identified and assigned to a vulnerability.",
- "required": [
- "type",
- "origin",
- "description"
- ],
- "properties": {
- "type": {
- "type": "string",
- "minLength": 1,
- "description": "Result of the scan.",
- "enum": [
- "flagged-as-likely-false-positive"
- ]
- },
- "origin": {
- "minLength": 1,
- "description": "Tool that issued the flag.",
- "type": "string"
- },
- "description": {
- "minLength": 1,
- "description": "What the flag is about.",
- "type": "string"
- }
- }
- }
- },
- "location": {
- "type": "object",
- "description": "Identifies the vulnerability's location.",
- "required": [
- "file",
- "dependency"
- ],
- "properties": {
- "file": {
- "type": "string",
- "minLength": 1,
- "description": "Path to the manifest or lock file where the dependency is declared (such as yarn.lock)."
- },
- "dependency": {
- "type": "object",
- "description": "Describes the dependency of a project where the vulnerability is located.",
- "properties": {
- "package": {
- "type": "object",
- "description": "Provides information on the package where the vulnerability is located.",
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the package where the vulnerability is located."
- }
- }
- },
- "version": {
- "type": "string",
- "description": "Version of the vulnerable package."
- },
- "iid": {
- "description": "ID that identifies the dependency in the scope of a dependency file.",
- "type": "number"
- },
- "direct": {
- "type": "boolean",
- "description": "Tells whether this is a direct, top-level dependency of the scanned project."
- },
- "dependency_path": {
- "type": "array",
- "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.",
- "items": {
- "type": "object",
- "required": [
- "iid"
- ],
- "properties": {
- "iid": {
- "type": "number",
- "description": "ID that is unique in the scope of a parent object, and specific to the resource type."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- }
- },
- "remediations": {
- "type": "array",
- "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
- "items": {
- "type": "object",
- "required": [
- "fixes",
- "summary",
- "diff"
- ],
- "properties": {
- "fixes": {
- "type": "array",
- "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
- "items": {
- "type": "object",
- "required": [
- "cve"
- ],
- "properties": {
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- }
- }
- }
- },
- "summary": {
- "type": "string",
- "minLength": 1,
- "description": "An overview of how the vulnerabilities were fixed."
- },
- "diff": {
- "type": "string",
- "minLength": 1,
- "description": "A base64-encoded remediation code diff, compatible with git apply."
- }
- }
- }
- },
- "dependency_files": {
- "type": "array",
- "description": "List of dependency files identified in the project.",
- "items": {
- "type": "object",
- "required": [
- "path",
- "package_manager",
- "dependencies"
- ],
- "properties": {
- "path": {
- "type": "string",
- "minLength": 1
- },
- "package_manager": {
- "type": "string",
- "minLength": 1
- },
- "dependencies": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Describes the dependency of a project where the vulnerability is located.",
- "properties": {
- "package": {
- "type": "object",
- "description": "Provides information on the package where the vulnerability is located.",
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the package where the vulnerability is located."
- }
- }
- },
- "version": {
- "type": "string",
- "description": "Version of the vulnerable package."
- },
- "iid": {
- "description": "ID that identifies the dependency in the scope of a dependency file.",
- "type": "number"
- },
- "direct": {
- "type": "boolean",
- "description": "Tells whether this is a direct, top-level dependency of the scanned project."
- },
- "dependency_path": {
- "type": "array",
- "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.",
- "items": {
- "type": "object",
- "required": [
- "iid"
- ],
- "properties": {
- "iid": {
- "type": "number",
- "description": "ID that is unique in the scope of a parent object, and specific to the resource type."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- }
- }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.5/sast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.5/sast-report-format.json
deleted file mode 100644
index b10b199a97c..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.5/sast-report-format.json
+++ /dev/null
@@ -1,869 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Report format for GitLab SAST",
- "description": "This schema provides the report format for Static Application Security Testing analyzers (https://docs.gitlab.com/ee/user/application_security/sast).",
- "definitions": {
- "detail_type": {
- "oneOf": [
- {
- "$ref": "#/definitions/named_list"
- },
- {
- "$ref": "#/definitions/list"
- },
- {
- "$ref": "#/definitions/table"
- },
- {
- "$ref": "#/definitions/text"
- },
- {
- "$ref": "#/definitions/url"
- },
- {
- "$ref": "#/definitions/code"
- },
- {
- "$ref": "#/definitions/value"
- },
- {
- "$ref": "#/definitions/diff"
- },
- {
- "$ref": "#/definitions/markdown"
- },
- {
- "$ref": "#/definitions/commit"
- },
- {
- "$ref": "#/definitions/file_location"
- },
- {
- "$ref": "#/definitions/module_location"
- }
- ]
- },
- "text_value": {
- "type": "string"
- },
- "named_field": {
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "$ref": "#/definitions/text_value",
- "minLength": 1
- },
- "description": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "named_list": {
- "type": "object",
- "description": "An object with named and typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "named-list"
- },
- "items": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- {
- "$ref": "#/definitions/named_field"
- },
- {
- "$ref": "#/definitions/detail_type"
- }
- ]
- }
- }
- }
- }
- },
- "list": {
- "type": "object",
- "description": "A list of typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- },
- "table": {
- "type": "object",
- "description": "A table of typed fields",
- "required": [
- "type",
- "rows"
- ],
- "properties": {
- "type": {
- "const": "table"
- },
- "header": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- },
- "rows": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- }
- },
- "text": {
- "type": "object",
- "description": "Raw text",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "text"
- },
- "value": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "url": {
- "type": "object",
- "description": "A single URL",
- "required": [
- "type",
- "href"
- ],
- "properties": {
- "type": {
- "const": "url"
- },
- "text": {
- "$ref": "#/definitions/text_value"
- },
- "href": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "http://mysite.com"
- ]
- }
- }
- },
- "code": {
- "type": "object",
- "description": "A codeblock",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "code"
- },
- "value": {
- "type": "string"
- },
- "lang": {
- "type": "string",
- "description": "A programming language"
- }
- }
- },
- "value": {
- "type": "object",
- "description": "A field that can store a range of types of value",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "value"
- },
- "value": {
- "type": [
- "number",
- "string",
- "boolean"
- ]
- }
- }
- },
- "diff": {
- "type": "object",
- "description": "A diff",
- "required": [
- "type",
- "before",
- "after"
- ],
- "properties": {
- "type": {
- "const": "diff"
- },
- "before": {
- "type": "string"
- },
- "after": {
- "type": "string"
- }
- }
- },
- "markdown": {
- "type": "object",
- "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "markdown"
- },
- "value": {
- "$ref": "#/definitions/text_value",
- "examples": [
- "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
- ]
- }
- }
- },
- "commit": {
- "type": "object",
- "description": "A commit/tag/branch within the GitLab project",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "commit"
- },
- "value": {
- "type": "string",
- "description": "The commit SHA",
- "minLength": 1
- }
- }
- },
- "file_location": {
- "type": "object",
- "description": "A location within a file in the project",
- "required": [
- "type",
- "file_name",
- "line_start"
- ],
- "properties": {
- "type": {
- "const": "file-location"
- },
- "file_name": {
- "type": "string",
- "minLength": 1
- },
- "line_start": {
- "type": "integer"
- },
- "line_end": {
- "type": "integer"
- }
- }
- },
- "module_location": {
- "type": "object",
- "description": "A location within a binary module of the form module+relative_offset",
- "required": [
- "type",
- "module_name",
- "offset"
- ],
- "properties": {
- "type": {
- "const": "module-location"
- },
- "module_name": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "compiled_binary"
- ]
- },
- "offset": {
- "type": "integer",
- "examples": [
- 100
- ]
- }
- }
- }
- },
- "self": {
- "version": "14.0.5"
- },
- "required": [
- "version",
- "vulnerabilities"
- ],
- "additionalProperties": true,
- "properties": {
- "scan": {
- "type": "object",
- "required": [
- "end_time",
- "scanner",
- "start_time",
- "status",
- "type"
- ],
- "properties": {
- "end_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-01-28T03:26:02"
- ]
- },
- "messages": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Communication intended for the initiator of a scan.",
- "required": [
- "level",
- "value"
- ],
- "properties": {
- "level": {
- "type": "string",
- "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
- "enum": [
- "info",
- "warn",
- "fatal"
- ],
- "examples": [
- "info"
- ]
- },
- "value": {
- "type": "string",
- "description": "The message to communicate.",
- "minLength": 1,
- "examples": [
- "Permission denied, scanning aborted"
- ]
- }
- }
- }
- },
- "analyzer": {
- "type": "object",
- "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the analyzer.",
- "minLength": 1,
- "examples": [
- "gitlab-dast"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the analyzer, not required to be unique.",
- "minLength": 1,
- "examples": [
- "GitLab DAST"
- ]
- },
- "url": {
- "type": "string",
- "format": "uri",
- "pattern": "^https?://.+",
- "description": "A link to more information about the analyzer.",
- "examples": [
- "https://docs.gitlab.com/ee/user/application_security/dast"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the analyzer.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- },
- "version": {
- "type": "string",
- "description": "The version of the analyzer.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- }
- }
- },
- "scanner": {
- "type": "object",
- "description": "Object defining the scanner used to perform the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the scanner.",
- "minLength": 1,
- "examples": [
- "my-sast-scanner"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the scanner, not required to be unique.",
- "minLength": 1,
- "examples": [
- "My SAST Scanner"
- ]
- },
- "url": {
- "type": "string",
- "description": "A link to more information about the scanner.",
- "examples": [
- "https://scanner.url"
- ]
- },
- "version": {
- "type": "string",
- "description": "The version of the scanner.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the scanner.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- }
- }
- },
- "start_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-02-14T16:01:59"
- ]
- },
- "status": {
- "type": "string",
- "description": "Result of the scan.",
- "enum": [
- "success",
- "failure"
- ]
- },
- "type": {
- "type": "string",
- "description": "Type of the scan.",
- "enum": [
- "sast"
- ]
- }
- }
- },
- "schema": {
- "type": "string",
- "description": "URI pointing to the validating security report schema.",
- "format": "uri"
- },
- "version": {
- "type": "string",
- "description": "The version of the schema to which the JSON report conforms.",
- "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
- },
- "vulnerabilities": {
- "type": "array",
- "description": "Array of vulnerability objects.",
- "items": {
- "type": "object",
- "description": "Describes the vulnerability.",
- "required": [
- "category",
- "cve",
- "identifiers",
- "location",
- "scanner"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
- "examples": [
- "642735a5-1425-428d-8d4e-3c854885a3c9"
- ]
- },
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
- "name": {
- "type": "string",
- "description": "The name of the vulnerability. This must not include the finding's specific information."
- },
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
- "description": {
- "type": "string",
- "description": "A long text section describing the vulnerability more fully."
- },
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
- "severity": {
- "type": "string",
- "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Info",
- "Unknown",
- "Low",
- "Medium",
- "High",
- "Critical"
- ]
- },
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
- "solution": {
- "type": "string",
- "description": "Explanation of how to fix the vulnerability."
- },
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
- "identifiers": {
- "type": "array",
- "minItems": 1,
- "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
- "items": {
- "type": "object",
- "required": [
- "type",
- "name",
- "value"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
- "minLength": 1
- },
- "name": {
- "type": "string",
- "description": "Human-readable name of the identifier.",
- "minLength": 1
- },
- "url": {
- "type": "string",
- "description": "URL of the identifier's documentation.",
- "format": "uri"
- },
- "value": {
- "type": "string",
- "description": "Value of the identifier, for matching purpose.",
- "minLength": 1
- }
- }
- }
- },
- "links": {
- "type": "array",
- "description": "An array of references to external documentation or articles that describe the vulnerability.",
- "items": {
- "type": "object",
- "required": [
- "url"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the vulnerability details link."
- },
- "url": {
- "type": "string",
- "description": "URL of the vulnerability details document.",
- "format": "uri"
- }
- }
- }
- },
- "details": {
- "$ref": "#/definitions/named_list/properties/items"
- },
- "tracking": {
- "description": "Describes how this vulnerability should be tracked as the project changes.",
- "oneOf": [
- {
- "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
- "required": [
- "items"
- ],
- "properties": {
- "type": {
- "const": "source"
- },
- "items": {
- "type": "array",
- "items": {
- "description": "An item that should be tracked using source-specific tracking methods.",
- "type": "object",
- "required": [
- "signatures"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the file that includes the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the file that includes the vulnerability."
- },
- "signatures": {
- "type": "array",
- "description": "An array of calculated tracking signatures for this tracking item.",
- "minItems": 1,
- "items": {
- "description": "A calculated tracking signature value and metadata.",
- "required": [
- "algorithm",
- "value"
- ],
- "properties": {
- "algorithm": {
- "type": "string",
- "description": "The algorithm used to generate the signature."
- },
- "value": {
- "type": "string",
- "description": "The result of this signature algorithm."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "Each tracking type must declare its own type."
- }
- }
- },
- "flags": {
- "description": "Flags that can be attached to vulnerabilities.",
- "type": "array",
- "items": {
- "type": "object",
- "description": "Informational flags identified and assigned to a vulnerability.",
- "required": [
- "type",
- "origin",
- "description"
- ],
- "properties": {
- "type": {
- "type": "string",
- "minLength": 1,
- "description": "Result of the scan.",
- "enum": [
- "flagged-as-likely-false-positive"
- ]
- },
- "origin": {
- "minLength": 1,
- "description": "Tool that issued the flag.",
- "type": "string"
- },
- "description": {
- "minLength": 1,
- "description": "What the flag is about.",
- "type": "string"
- }
- }
- }
- },
- "location": {
- "type": "object",
- "description": "Identifies the vulnerability's location.",
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the code affected by the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the code affected by the vulnerability."
- },
- "class": {
- "type": "string",
- "description": "Provides the name of the class where the vulnerability is located."
- },
- "method": {
- "type": "string",
- "description": "Provides the name of the method where the vulnerability is located."
- }
- }
- },
- "raw_source_code_extract": {
- "type": "string",
- "description": "Provides an unsanitized excerpt of the affected source code."
- }
- }
- }
- },
- "remediations": {
- "type": "array",
- "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
- "items": {
- "type": "object",
- "required": [
- "fixes",
- "summary",
- "diff"
- ],
- "properties": {
- "fixes": {
- "type": "array",
- "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
- "items": {
- "type": "object",
- "required": [
- "cve"
- ],
- "properties": {
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- }
- }
- }
- },
- "summary": {
- "type": "string",
- "minLength": 1,
- "description": "An overview of how the vulnerabilities were fixed."
- },
- "diff": {
- "type": "string",
- "minLength": 1,
- "description": "A base64-encoded remediation code diff, compatible with git apply."
- }
- }
- }
- }
- }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.5/secret-detection-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.5/secret-detection-report-format.json
deleted file mode 100644
index 5bd945c8ab5..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.5/secret-detection-report-format.json
+++ /dev/null
@@ -1,892 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Report format for GitLab Secret Detection",
- "description": "This schema provides the the report format for the Secret Detection analyzer (https://docs.gitlab.com/ee/user/application_security/secret_detection)",
- "definitions": {
- "detail_type": {
- "oneOf": [
- {
- "$ref": "#/definitions/named_list"
- },
- {
- "$ref": "#/definitions/list"
- },
- {
- "$ref": "#/definitions/table"
- },
- {
- "$ref": "#/definitions/text"
- },
- {
- "$ref": "#/definitions/url"
- },
- {
- "$ref": "#/definitions/code"
- },
- {
- "$ref": "#/definitions/value"
- },
- {
- "$ref": "#/definitions/diff"
- },
- {
- "$ref": "#/definitions/markdown"
- },
- {
- "$ref": "#/definitions/commit"
- },
- {
- "$ref": "#/definitions/file_location"
- },
- {
- "$ref": "#/definitions/module_location"
- }
- ]
- },
- "text_value": {
- "type": "string"
- },
- "named_field": {
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "$ref": "#/definitions/text_value",
- "minLength": 1
- },
- "description": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "named_list": {
- "type": "object",
- "description": "An object with named and typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "named-list"
- },
- "items": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- {
- "$ref": "#/definitions/named_field"
- },
- {
- "$ref": "#/definitions/detail_type"
- }
- ]
- }
- }
- }
- }
- },
- "list": {
- "type": "object",
- "description": "A list of typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- },
- "table": {
- "type": "object",
- "description": "A table of typed fields",
- "required": [
- "type",
- "rows"
- ],
- "properties": {
- "type": {
- "const": "table"
- },
- "header": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- },
- "rows": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- }
- },
- "text": {
- "type": "object",
- "description": "Raw text",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "text"
- },
- "value": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "url": {
- "type": "object",
- "description": "A single URL",
- "required": [
- "type",
- "href"
- ],
- "properties": {
- "type": {
- "const": "url"
- },
- "text": {
- "$ref": "#/definitions/text_value"
- },
- "href": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "http://mysite.com"
- ]
- }
- }
- },
- "code": {
- "type": "object",
- "description": "A codeblock",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "code"
- },
- "value": {
- "type": "string"
- },
- "lang": {
- "type": "string",
- "description": "A programming language"
- }
- }
- },
- "value": {
- "type": "object",
- "description": "A field that can store a range of types of value",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "value"
- },
- "value": {
- "type": [
- "number",
- "string",
- "boolean"
- ]
- }
- }
- },
- "diff": {
- "type": "object",
- "description": "A diff",
- "required": [
- "type",
- "before",
- "after"
- ],
- "properties": {
- "type": {
- "const": "diff"
- },
- "before": {
- "type": "string"
- },
- "after": {
- "type": "string"
- }
- }
- },
- "markdown": {
- "type": "object",
- "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "markdown"
- },
- "value": {
- "$ref": "#/definitions/text_value",
- "examples": [
- "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
- ]
- }
- }
- },
- "commit": {
- "type": "object",
- "description": "A commit/tag/branch within the GitLab project",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "commit"
- },
- "value": {
- "type": "string",
- "description": "The commit SHA",
- "minLength": 1
- }
- }
- },
- "file_location": {
- "type": "object",
- "description": "A location within a file in the project",
- "required": [
- "type",
- "file_name",
- "line_start"
- ],
- "properties": {
- "type": {
- "const": "file-location"
- },
- "file_name": {
- "type": "string",
- "minLength": 1
- },
- "line_start": {
- "type": "integer"
- },
- "line_end": {
- "type": "integer"
- }
- }
- },
- "module_location": {
- "type": "object",
- "description": "A location within a binary module of the form module+relative_offset",
- "required": [
- "type",
- "module_name",
- "offset"
- ],
- "properties": {
- "type": {
- "const": "module-location"
- },
- "module_name": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "compiled_binary"
- ]
- },
- "offset": {
- "type": "integer",
- "examples": [
- 100
- ]
- }
- }
- }
- },
- "self": {
- "version": "14.0.5"
- },
- "required": [
- "version",
- "vulnerabilities"
- ],
- "additionalProperties": true,
- "properties": {
- "scan": {
- "type": "object",
- "required": [
- "end_time",
- "scanner",
- "start_time",
- "status",
- "type"
- ],
- "properties": {
- "end_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-01-28T03:26:02"
- ]
- },
- "messages": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Communication intended for the initiator of a scan.",
- "required": [
- "level",
- "value"
- ],
- "properties": {
- "level": {
- "type": "string",
- "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
- "enum": [
- "info",
- "warn",
- "fatal"
- ],
- "examples": [
- "info"
- ]
- },
- "value": {
- "type": "string",
- "description": "The message to communicate.",
- "minLength": 1,
- "examples": [
- "Permission denied, scanning aborted"
- ]
- }
- }
- }
- },
- "analyzer": {
- "type": "object",
- "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the analyzer.",
- "minLength": 1,
- "examples": [
- "gitlab-dast"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the analyzer, not required to be unique.",
- "minLength": 1,
- "examples": [
- "GitLab DAST"
- ]
- },
- "url": {
- "type": "string",
- "format": "uri",
- "pattern": "^https?://.+",
- "description": "A link to more information about the analyzer.",
- "examples": [
- "https://docs.gitlab.com/ee/user/application_security/dast"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the analyzer.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- },
- "version": {
- "type": "string",
- "description": "The version of the analyzer.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- }
- }
- },
- "scanner": {
- "type": "object",
- "description": "Object defining the scanner used to perform the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the scanner.",
- "minLength": 1,
- "examples": [
- "my-sast-scanner"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the scanner, not required to be unique.",
- "minLength": 1,
- "examples": [
- "My SAST Scanner"
- ]
- },
- "url": {
- "type": "string",
- "description": "A link to more information about the scanner.",
- "examples": [
- "https://scanner.url"
- ]
- },
- "version": {
- "type": "string",
- "description": "The version of the scanner.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the scanner.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- }
- }
- },
- "start_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-02-14T16:01:59"
- ]
- },
- "status": {
- "type": "string",
- "description": "Result of the scan.",
- "enum": [
- "success",
- "failure"
- ]
- },
- "type": {
- "type": "string",
- "description": "Type of the scan.",
- "enum": [
- "secret_detection"
- ]
- }
- }
- },
- "schema": {
- "type": "string",
- "description": "URI pointing to the validating security report schema.",
- "format": "uri"
- },
- "version": {
- "type": "string",
- "description": "The version of the schema to which the JSON report conforms.",
- "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
- },
- "vulnerabilities": {
- "type": "array",
- "description": "Array of vulnerability objects.",
- "items": {
- "type": "object",
- "description": "Describes the vulnerability.",
- "required": [
- "category",
- "cve",
- "identifiers",
- "location",
- "scanner"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
- "examples": [
- "642735a5-1425-428d-8d4e-3c854885a3c9"
- ]
- },
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
- "name": {
- "type": "string",
- "description": "The name of the vulnerability. This must not include the finding's specific information."
- },
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
- "description": {
- "type": "string",
- "description": "A long text section describing the vulnerability more fully."
- },
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
- "severity": {
- "type": "string",
- "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Info",
- "Unknown",
- "Low",
- "Medium",
- "High",
- "Critical"
- ]
- },
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
- "solution": {
- "type": "string",
- "description": "Explanation of how to fix the vulnerability."
- },
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
- "identifiers": {
- "type": "array",
- "minItems": 1,
- "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
- "items": {
- "type": "object",
- "required": [
- "type",
- "name",
- "value"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
- "minLength": 1
- },
- "name": {
- "type": "string",
- "description": "Human-readable name of the identifier.",
- "minLength": 1
- },
- "url": {
- "type": "string",
- "description": "URL of the identifier's documentation.",
- "format": "uri"
- },
- "value": {
- "type": "string",
- "description": "Value of the identifier, for matching purpose.",
- "minLength": 1
- }
- }
- }
- },
- "links": {
- "type": "array",
- "description": "An array of references to external documentation or articles that describe the vulnerability.",
- "items": {
- "type": "object",
- "required": [
- "url"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the vulnerability details link."
- },
- "url": {
- "type": "string",
- "description": "URL of the vulnerability details document.",
- "format": "uri"
- }
- }
- }
- },
- "details": {
- "$ref": "#/definitions/named_list/properties/items"
- },
- "tracking": {
- "description": "Describes how this vulnerability should be tracked as the project changes.",
- "oneOf": [
- {
- "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
- "required": [
- "items"
- ],
- "properties": {
- "type": {
- "const": "source"
- },
- "items": {
- "type": "array",
- "items": {
- "description": "An item that should be tracked using source-specific tracking methods.",
- "type": "object",
- "required": [
- "signatures"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the file that includes the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the file that includes the vulnerability."
- },
- "signatures": {
- "type": "array",
- "description": "An array of calculated tracking signatures for this tracking item.",
- "minItems": 1,
- "items": {
- "description": "A calculated tracking signature value and metadata.",
- "required": [
- "algorithm",
- "value"
- ],
- "properties": {
- "algorithm": {
- "type": "string",
- "description": "The algorithm used to generate the signature."
- },
- "value": {
- "type": "string",
- "description": "The result of this signature algorithm."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "Each tracking type must declare its own type."
- }
- }
- },
- "flags": {
- "description": "Flags that can be attached to vulnerabilities.",
- "type": "array",
- "items": {
- "type": "object",
- "description": "Informational flags identified and assigned to a vulnerability.",
- "required": [
- "type",
- "origin",
- "description"
- ],
- "properties": {
- "type": {
- "type": "string",
- "minLength": 1,
- "description": "Result of the scan.",
- "enum": [
- "flagged-as-likely-false-positive"
- ]
- },
- "origin": {
- "minLength": 1,
- "description": "Tool that issued the flag.",
- "type": "string"
- },
- "description": {
- "minLength": 1,
- "description": "What the flag is about.",
- "type": "string"
- }
- }
- }
- },
- "location": {
- "required": [
- "commit"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located"
- },
- "commit": {
- "type": "object",
- "description": "Represents the commit in which the vulnerability was detected",
- "required": [
- "sha"
- ],
- "properties": {
- "author": {
- "type": "string"
- },
- "date": {
- "type": "string"
- },
- "message": {
- "type": "string"
- },
- "sha": {
- "type": "string",
- "minLength": 1
- }
- }
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the code affected by the vulnerability"
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the code affected by the vulnerability"
- },
- "class": {
- "type": "string",
- "description": "Provides the name of the class where the vulnerability is located"
- },
- "method": {
- "type": "string",
- "description": "Provides the name of the method where the vulnerability is located"
- }
- }
- },
- "raw_source_code_extract": {
- "type": "string",
- "description": "Provides an unsanitized excerpt of the affected source code."
- }
- }
- }
- },
- "remediations": {
- "type": "array",
- "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
- "items": {
- "type": "object",
- "required": [
- "fixes",
- "summary",
- "diff"
- ],
- "properties": {
- "fixes": {
- "type": "array",
- "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
- "items": {
- "type": "object",
- "required": [
- "cve"
- ],
- "properties": {
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- }
- }
- }
- },
- "summary": {
- "type": "string",
- "minLength": 1,
- "description": "An overview of how the vulnerabilities were fixed."
- },
- "diff": {
- "type": "string",
- "minLength": 1,
- "description": "A base64-encoded remediation code diff, compatible with git apply."
- }
- }
- }
- }
- }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.6/cluster-image-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.6/cluster-image-scanning-report-format.json
deleted file mode 100644
index 951b0fea013..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.6/cluster-image-scanning-report-format.json
+++ /dev/null
@@ -1,977 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Report format for GitLab Cluster Image Scanning",
- "description": "This schema provides the the report format for Cluster Image Scanning (https://docs.gitlab.com/ee/user/application_security/cluster_image_scanning/).",
- "definitions": {
- "detail_type": {
- "oneOf": [
- {
- "$ref": "#/definitions/named_list"
- },
- {
- "$ref": "#/definitions/list"
- },
- {
- "$ref": "#/definitions/table"
- },
- {
- "$ref": "#/definitions/text"
- },
- {
- "$ref": "#/definitions/url"
- },
- {
- "$ref": "#/definitions/code"
- },
- {
- "$ref": "#/definitions/value"
- },
- {
- "$ref": "#/definitions/diff"
- },
- {
- "$ref": "#/definitions/markdown"
- },
- {
- "$ref": "#/definitions/commit"
- },
- {
- "$ref": "#/definitions/file_location"
- },
- {
- "$ref": "#/definitions/module_location"
- }
- ]
- },
- "text_value": {
- "type": "string"
- },
- "named_field": {
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "$ref": "#/definitions/text_value",
- "minLength": 1
- },
- "description": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "named_list": {
- "type": "object",
- "description": "An object with named and typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "named-list"
- },
- "items": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- {
- "$ref": "#/definitions/named_field"
- },
- {
- "$ref": "#/definitions/detail_type"
- }
- ]
- }
- }
- }
- }
- },
- "list": {
- "type": "object",
- "description": "A list of typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- },
- "table": {
- "type": "object",
- "description": "A table of typed fields",
- "required": [
- "type",
- "rows"
- ],
- "properties": {
- "type": {
- "const": "table"
- },
- "header": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- },
- "rows": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- }
- },
- "text": {
- "type": "object",
- "description": "Raw text",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "text"
- },
- "value": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "url": {
- "type": "object",
- "description": "A single URL",
- "required": [
- "type",
- "href"
- ],
- "properties": {
- "type": {
- "const": "url"
- },
- "text": {
- "$ref": "#/definitions/text_value"
- },
- "href": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "http://mysite.com"
- ]
- }
- }
- },
- "code": {
- "type": "object",
- "description": "A codeblock",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "code"
- },
- "value": {
- "type": "string"
- },
- "lang": {
- "type": "string",
- "description": "A programming language"
- }
- }
- },
- "value": {
- "type": "object",
- "description": "A field that can store a range of types of value",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "value"
- },
- "value": {
- "type": [
- "number",
- "string",
- "boolean"
- ]
- }
- }
- },
- "diff": {
- "type": "object",
- "description": "A diff",
- "required": [
- "type",
- "before",
- "after"
- ],
- "properties": {
- "type": {
- "const": "diff"
- },
- "before": {
- "type": "string"
- },
- "after": {
- "type": "string"
- }
- }
- },
- "markdown": {
- "type": "object",
- "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "markdown"
- },
- "value": {
- "$ref": "#/definitions/text_value",
- "examples": [
- "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
- ]
- }
- }
- },
- "commit": {
- "type": "object",
- "description": "A commit/tag/branch within the GitLab project",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "commit"
- },
- "value": {
- "type": "string",
- "description": "The commit SHA",
- "minLength": 1
- }
- }
- },
- "file_location": {
- "type": "object",
- "description": "A location within a file in the project",
- "required": [
- "type",
- "file_name",
- "line_start"
- ],
- "properties": {
- "type": {
- "const": "file-location"
- },
- "file_name": {
- "type": "string",
- "minLength": 1
- },
- "line_start": {
- "type": "integer"
- },
- "line_end": {
- "type": "integer"
- }
- }
- },
- "module_location": {
- "type": "object",
- "description": "A location within a binary module of the form module+relative_offset",
- "required": [
- "type",
- "module_name",
- "offset"
- ],
- "properties": {
- "type": {
- "const": "module-location"
- },
- "module_name": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "compiled_binary"
- ]
- },
- "offset": {
- "type": "integer",
- "examples": [
- 100
- ]
- }
- }
- }
- },
- "self": {
- "version": "14.0.6"
- },
- "required": [
- "version",
- "vulnerabilities"
- ],
- "additionalProperties": true,
- "properties": {
- "scan": {
- "type": "object",
- "required": [
- "end_time",
- "scanner",
- "start_time",
- "status",
- "type"
- ],
- "properties": {
- "end_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-01-28T03:26:02"
- ]
- },
- "messages": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Communication intended for the initiator of a scan.",
- "required": [
- "level",
- "value"
- ],
- "properties": {
- "level": {
- "type": "string",
- "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
- "enum": [
- "info",
- "warn",
- "fatal"
- ],
- "examples": [
- "info"
- ]
- },
- "value": {
- "type": "string",
- "description": "The message to communicate.",
- "minLength": 1,
- "examples": [
- "Permission denied, scanning aborted"
- ]
- }
- }
- }
- },
- "analyzer": {
- "type": "object",
- "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the analyzer.",
- "minLength": 1,
- "examples": [
- "gitlab-dast"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the analyzer, not required to be unique.",
- "minLength": 1,
- "examples": [
- "GitLab DAST"
- ]
- },
- "url": {
- "type": "string",
- "format": "uri",
- "pattern": "^https?://.+",
- "description": "A link to more information about the analyzer.",
- "examples": [
- "https://docs.gitlab.com/ee/user/application_security/dast"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the analyzer.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- },
- "version": {
- "type": "string",
- "description": "The version of the analyzer.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- }
- }
- },
- "scanner": {
- "type": "object",
- "description": "Object defining the scanner used to perform the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the scanner.",
- "minLength": 1,
- "examples": [
- "my-sast-scanner"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the scanner, not required to be unique.",
- "minLength": 1,
- "examples": [
- "My SAST Scanner"
- ]
- },
- "url": {
- "type": "string",
- "description": "A link to more information about the scanner.",
- "examples": [
- "https://scanner.url"
- ]
- },
- "version": {
- "type": "string",
- "description": "The version of the scanner.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the scanner.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- }
- }
- },
- "start_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-02-14T16:01:59"
- ]
- },
- "status": {
- "type": "string",
- "description": "Result of the scan.",
- "enum": [
- "success",
- "failure"
- ]
- },
- "type": {
- "type": "string",
- "description": "Type of the scan.",
- "enum": [
- "cluster_image_scanning"
- ]
- }
- }
- },
- "schema": {
- "type": "string",
- "description": "URI pointing to the validating security report schema.",
- "format": "uri"
- },
- "version": {
- "type": "string",
- "description": "The version of the schema to which the JSON report conforms.",
- "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
- },
- "vulnerabilities": {
- "type": "array",
- "description": "Array of vulnerability objects.",
- "items": {
- "type": "object",
- "description": "Describes the vulnerability using GitLab Flavored Markdown",
- "required": [
- "category",
- "cve",
- "identifiers",
- "location",
- "scanner"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
- "examples": [
- "642735a5-1425-428d-8d4e-3c854885a3c9"
- ]
- },
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
- "name": {
- "type": "string",
- "description": "The name of the vulnerability. This must not include the finding's specific information."
- },
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
- "description": {
- "type": "string",
- "description": "A long text section describing the vulnerability more fully."
- },
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
- "severity": {
- "type": "string",
- "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Info",
- "Unknown",
- "Low",
- "Medium",
- "High",
- "Critical"
- ]
- },
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
- "solution": {
- "type": "string",
- "description": "Explanation of how to fix the vulnerability."
- },
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
- "identifiers": {
- "type": "array",
- "minItems": 1,
- "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
- "items": {
- "type": "object",
- "required": [
- "type",
- "name",
- "value"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
- "minLength": 1
- },
- "name": {
- "type": "string",
- "description": "Human-readable name of the identifier.",
- "minLength": 1
- },
- "url": {
- "type": "string",
- "description": "URL of the identifier's documentation.",
- "format": "uri"
- },
- "value": {
- "type": "string",
- "description": "Value of the identifier, for matching purpose.",
- "minLength": 1
- }
- }
- }
- },
- "links": {
- "type": "array",
- "description": "An array of references to external documentation or articles that describe the vulnerability.",
- "items": {
- "type": "object",
- "required": [
- "url"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the vulnerability details link."
- },
- "url": {
- "type": "string",
- "description": "URL of the vulnerability details document.",
- "format": "uri"
- }
- }
- }
- },
- "details": {
- "$ref": "#/definitions/named_list/properties/items"
- },
- "tracking": {
- "description": "Describes how this vulnerability should be tracked as the project changes.",
- "oneOf": [
- {
- "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
- "required": [
- "items"
- ],
- "properties": {
- "type": {
- "const": "source"
- },
- "items": {
- "type": "array",
- "items": {
- "description": "An item that should be tracked using source-specific tracking methods.",
- "type": "object",
- "required": [
- "signatures"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the file that includes the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the file that includes the vulnerability."
- },
- "signatures": {
- "type": "array",
- "description": "An array of calculated tracking signatures for this tracking item.",
- "minItems": 1,
- "items": {
- "description": "A calculated tracking signature value and metadata.",
- "required": [
- "algorithm",
- "value"
- ],
- "properties": {
- "algorithm": {
- "type": "string",
- "description": "The algorithm used to generate the signature."
- },
- "value": {
- "type": "string",
- "description": "The result of this signature algorithm."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "Each tracking type must declare its own type."
- }
- }
- },
- "flags": {
- "description": "Flags that can be attached to vulnerabilities.",
- "type": "array",
- "items": {
- "type": "object",
- "description": "Informational flags identified and assigned to a vulnerability.",
- "required": [
- "type",
- "origin",
- "description"
- ],
- "properties": {
- "type": {
- "type": "string",
- "minLength": 1,
- "description": "Result of the scan.",
- "enum": [
- "flagged-as-likely-false-positive"
- ]
- },
- "origin": {
- "minLength": 1,
- "description": "Tool that issued the flag.",
- "type": "string"
- },
- "description": {
- "minLength": 1,
- "description": "What the flag is about.",
- "type": "string"
- }
- }
- }
- },
- "location": {
- "type": "object",
- "description": "Identifies the vulnerability's location.",
- "required": [
- "dependency",
- "image",
- "kubernetes_resource"
- ],
- "properties": {
- "dependency": {
- "type": "object",
- "description": "Describes the dependency of a project where the vulnerability is located.",
- "properties": {
- "package": {
- "type": "object",
- "description": "Provides information on the package where the vulnerability is located.",
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the package where the vulnerability is located."
- }
- }
- },
- "version": {
- "type": "string",
- "description": "Version of the vulnerable package."
- },
- "iid": {
- "description": "ID that identifies the dependency in the scope of a dependency file.",
- "type": "number"
- },
- "direct": {
- "type": "boolean",
- "description": "Tells whether this is a direct, top-level dependency of the scanned project."
- },
- "dependency_path": {
- "type": "array",
- "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.",
- "items": {
- "type": "object",
- "required": [
- "iid"
- ],
- "properties": {
- "iid": {
- "type": "number",
- "description": "ID that is unique in the scope of a parent object, and specific to the resource type."
- }
- }
- }
- }
- }
- },
- "operating_system": {
- "type": "string",
- "minLength": 1,
- "maxLength": 255,
- "description": "The operating system that contains the vulnerable package."
- },
- "image": {
- "type": "string",
- "minLength": 1,
- "description": "The analyzed Docker image.",
- "examples": [
- "index.docker.io/library/nginx:1.21"
- ]
- },
- "kubernetes_resource": {
- "type": "object",
- "description": "The specific Kubernetes resource that was scanned.",
- "required": [
- "namespace",
- "kind",
- "name",
- "container_name"
- ],
- "properties": {
- "namespace": {
- "type": "string",
- "minLength": 1,
- "maxLength": 255,
- "description": "The Kubernetes namespace the resource that had its image scanned.",
- "examples": [
- "default",
- "staging",
- "production"
- ]
- },
- "kind": {
- "type": "string",
- "minLength": 1,
- "maxLength": 255,
- "description": "The Kubernetes kind the resource that had its image scanned.",
- "examples": [
- "Deployment",
- "DaemonSet"
- ]
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "maxLength": 255,
- "description": "The name of the resource that had its image scanned.",
- "examples": [
- "nginx-ingress"
- ]
- },
- "container_name": {
- "type": "string",
- "minLength": 1,
- "maxLength": 255,
- "description": "The name of the container that had its image scanned.",
- "examples": [
- "nginx"
- ]
- },
- "agent_id": {
- "type": "string",
- "minLength": 1,
- "maxLength": 255,
- "description": "The GitLab ID of the Kubernetes Agent which performed the scan.",
- "examples": [
- "1234"
- ]
- },
- "cluster_id": {
- "type": "string",
- "minLength": 1,
- "maxLength": 255,
- "description": "The GitLab ID of the Kubernetes cluster when using cluster integration.",
- "examples": [
- "1234"
- ]
- }
- }
- }
- }
- }
- }
- }
- },
- "remediations": {
- "type": "array",
- "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
- "items": {
- "type": "object",
- "required": [
- "fixes",
- "summary",
- "diff"
- ],
- "properties": {
- "fixes": {
- "type": "array",
- "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
- "items": {
- "type": "object",
- "required": [
- "cve"
- ],
- "properties": {
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- }
- }
- }
- },
- "summary": {
- "type": "string",
- "minLength": 1,
- "description": "An overview of how the vulnerabilities were fixed."
- },
- "diff": {
- "type": "string",
- "minLength": 1,
- "description": "A base64-encoded remediation code diff, compatible with git apply."
- }
- }
- }
- }
- }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.6/coverage-fuzzing-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.6/coverage-fuzzing-report-format.json
deleted file mode 100644
index de79d4b52ab..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.6/coverage-fuzzing-report-format.json
+++ /dev/null
@@ -1,874 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Report format for GitLab Fuzz Testing",
- "description": "This schema provides the report format for Coverage Guided Fuzz Testing (https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing).",
- "definitions": {
- "detail_type": {
- "oneOf": [
- {
- "$ref": "#/definitions/named_list"
- },
- {
- "$ref": "#/definitions/list"
- },
- {
- "$ref": "#/definitions/table"
- },
- {
- "$ref": "#/definitions/text"
- },
- {
- "$ref": "#/definitions/url"
- },
- {
- "$ref": "#/definitions/code"
- },
- {
- "$ref": "#/definitions/value"
- },
- {
- "$ref": "#/definitions/diff"
- },
- {
- "$ref": "#/definitions/markdown"
- },
- {
- "$ref": "#/definitions/commit"
- },
- {
- "$ref": "#/definitions/file_location"
- },
- {
- "$ref": "#/definitions/module_location"
- }
- ]
- },
- "text_value": {
- "type": "string"
- },
- "named_field": {
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "$ref": "#/definitions/text_value",
- "minLength": 1
- },
- "description": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "named_list": {
- "type": "object",
- "description": "An object with named and typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "named-list"
- },
- "items": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- {
- "$ref": "#/definitions/named_field"
- },
- {
- "$ref": "#/definitions/detail_type"
- }
- ]
- }
- }
- }
- }
- },
- "list": {
- "type": "object",
- "description": "A list of typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- },
- "table": {
- "type": "object",
- "description": "A table of typed fields",
- "required": [
- "type",
- "rows"
- ],
- "properties": {
- "type": {
- "const": "table"
- },
- "header": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- },
- "rows": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- }
- },
- "text": {
- "type": "object",
- "description": "Raw text",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "text"
- },
- "value": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "url": {
- "type": "object",
- "description": "A single URL",
- "required": [
- "type",
- "href"
- ],
- "properties": {
- "type": {
- "const": "url"
- },
- "text": {
- "$ref": "#/definitions/text_value"
- },
- "href": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "http://mysite.com"
- ]
- }
- }
- },
- "code": {
- "type": "object",
- "description": "A codeblock",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "code"
- },
- "value": {
- "type": "string"
- },
- "lang": {
- "type": "string",
- "description": "A programming language"
- }
- }
- },
- "value": {
- "type": "object",
- "description": "A field that can store a range of types of value",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "value"
- },
- "value": {
- "type": [
- "number",
- "string",
- "boolean"
- ]
- }
- }
- },
- "diff": {
- "type": "object",
- "description": "A diff",
- "required": [
- "type",
- "before",
- "after"
- ],
- "properties": {
- "type": {
- "const": "diff"
- },
- "before": {
- "type": "string"
- },
- "after": {
- "type": "string"
- }
- }
- },
- "markdown": {
- "type": "object",
- "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "markdown"
- },
- "value": {
- "$ref": "#/definitions/text_value",
- "examples": [
- "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
- ]
- }
- }
- },
- "commit": {
- "type": "object",
- "description": "A commit/tag/branch within the GitLab project",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "commit"
- },
- "value": {
- "type": "string",
- "description": "The commit SHA",
- "minLength": 1
- }
- }
- },
- "file_location": {
- "type": "object",
- "description": "A location within a file in the project",
- "required": [
- "type",
- "file_name",
- "line_start"
- ],
- "properties": {
- "type": {
- "const": "file-location"
- },
- "file_name": {
- "type": "string",
- "minLength": 1
- },
- "line_start": {
- "type": "integer"
- },
- "line_end": {
- "type": "integer"
- }
- }
- },
- "module_location": {
- "type": "object",
- "description": "A location within a binary module of the form module+relative_offset",
- "required": [
- "type",
- "module_name",
- "offset"
- ],
- "properties": {
- "type": {
- "const": "module-location"
- },
- "module_name": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "compiled_binary"
- ]
- },
- "offset": {
- "type": "integer",
- "examples": [
- 100
- ]
- }
- }
- }
- },
- "self": {
- "version": "14.0.6"
- },
- "required": [
- "version",
- "vulnerabilities"
- ],
- "additionalProperties": true,
- "properties": {
- "scan": {
- "type": "object",
- "required": [
- "end_time",
- "scanner",
- "start_time",
- "status",
- "type"
- ],
- "properties": {
- "end_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-01-28T03:26:02"
- ]
- },
- "messages": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Communication intended for the initiator of a scan.",
- "required": [
- "level",
- "value"
- ],
- "properties": {
- "level": {
- "type": "string",
- "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
- "enum": [
- "info",
- "warn",
- "fatal"
- ],
- "examples": [
- "info"
- ]
- },
- "value": {
- "type": "string",
- "description": "The message to communicate.",
- "minLength": 1,
- "examples": [
- "Permission denied, scanning aborted"
- ]
- }
- }
- }
- },
- "analyzer": {
- "type": "object",
- "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the analyzer.",
- "minLength": 1,
- "examples": [
- "gitlab-dast"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the analyzer, not required to be unique.",
- "minLength": 1,
- "examples": [
- "GitLab DAST"
- ]
- },
- "url": {
- "type": "string",
- "format": "uri",
- "pattern": "^https?://.+",
- "description": "A link to more information about the analyzer.",
- "examples": [
- "https://docs.gitlab.com/ee/user/application_security/dast"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the analyzer.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- },
- "version": {
- "type": "string",
- "description": "The version of the analyzer.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- }
- }
- },
- "scanner": {
- "type": "object",
- "description": "Object defining the scanner used to perform the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the scanner.",
- "minLength": 1,
- "examples": [
- "my-sast-scanner"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the scanner, not required to be unique.",
- "minLength": 1,
- "examples": [
- "My SAST Scanner"
- ]
- },
- "url": {
- "type": "string",
- "description": "A link to more information about the scanner.",
- "examples": [
- "https://scanner.url"
- ]
- },
- "version": {
- "type": "string",
- "description": "The version of the scanner.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the scanner.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- }
- }
- },
- "start_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-02-14T16:01:59"
- ]
- },
- "status": {
- "type": "string",
- "description": "Result of the scan.",
- "enum": [
- "success",
- "failure"
- ]
- },
- "type": {
- "type": "string",
- "description": "Type of the scan.",
- "enum": [
- "coverage_fuzzing"
- ]
- }
- }
- },
- "schema": {
- "type": "string",
- "description": "URI pointing to the validating security report schema.",
- "format": "uri"
- },
- "version": {
- "type": "string",
- "description": "The version of the schema to which the JSON report conforms.",
- "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
- },
- "vulnerabilities": {
- "type": "array",
- "description": "Array of vulnerability objects.",
- "items": {
- "type": "object",
- "description": "Describes the vulnerability using GitLab Flavored Markdown",
- "required": [
- "category",
- "cve",
- "identifiers",
- "location",
- "scanner"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
- "examples": [
- "642735a5-1425-428d-8d4e-3c854885a3c9"
- ]
- },
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
- "name": {
- "type": "string",
- "description": "The name of the vulnerability. This must not include the finding's specific information."
- },
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
- "description": {
- "type": "string",
- "description": "A long text section describing the vulnerability more fully."
- },
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
- "severity": {
- "type": "string",
- "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Info",
- "Unknown",
- "Low",
- "Medium",
- "High",
- "Critical"
- ]
- },
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
- "solution": {
- "type": "string",
- "description": "Explanation of how to fix the vulnerability."
- },
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
- "identifiers": {
- "type": "array",
- "minItems": 1,
- "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
- "items": {
- "type": "object",
- "required": [
- "type",
- "name",
- "value"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
- "minLength": 1
- },
- "name": {
- "type": "string",
- "description": "Human-readable name of the identifier.",
- "minLength": 1
- },
- "url": {
- "type": "string",
- "description": "URL of the identifier's documentation.",
- "format": "uri"
- },
- "value": {
- "type": "string",
- "description": "Value of the identifier, for matching purpose.",
- "minLength": 1
- }
- }
- }
- },
- "links": {
- "type": "array",
- "description": "An array of references to external documentation or articles that describe the vulnerability.",
- "items": {
- "type": "object",
- "required": [
- "url"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the vulnerability details link."
- },
- "url": {
- "type": "string",
- "description": "URL of the vulnerability details document.",
- "format": "uri"
- }
- }
- }
- },
- "details": {
- "$ref": "#/definitions/named_list/properties/items"
- },
- "tracking": {
- "description": "Describes how this vulnerability should be tracked as the project changes.",
- "oneOf": [
- {
- "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
- "required": [
- "items"
- ],
- "properties": {
- "type": {
- "const": "source"
- },
- "items": {
- "type": "array",
- "items": {
- "description": "An item that should be tracked using source-specific tracking methods.",
- "type": "object",
- "required": [
- "signatures"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the file that includes the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the file that includes the vulnerability."
- },
- "signatures": {
- "type": "array",
- "description": "An array of calculated tracking signatures for this tracking item.",
- "minItems": 1,
- "items": {
- "description": "A calculated tracking signature value and metadata.",
- "required": [
- "algorithm",
- "value"
- ],
- "properties": {
- "algorithm": {
- "type": "string",
- "description": "The algorithm used to generate the signature."
- },
- "value": {
- "type": "string",
- "description": "The result of this signature algorithm."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "Each tracking type must declare its own type."
- }
- }
- },
- "flags": {
- "description": "Flags that can be attached to vulnerabilities.",
- "type": "array",
- "items": {
- "type": "object",
- "description": "Informational flags identified and assigned to a vulnerability.",
- "required": [
- "type",
- "origin",
- "description"
- ],
- "properties": {
- "type": {
- "type": "string",
- "minLength": 1,
- "description": "Result of the scan.",
- "enum": [
- "flagged-as-likely-false-positive"
- ]
- },
- "origin": {
- "minLength": 1,
- "description": "Tool that issued the flag.",
- "type": "string"
- },
- "description": {
- "minLength": 1,
- "description": "What the flag is about.",
- "type": "string"
- }
- }
- }
- },
- "location": {
- "description": "The location of the error",
- "type": "object",
- "properties": {
- "crash_address": {
- "type": "string",
- "description": "The relative address in memory were the crash occurred.",
- "examples": [
- "0xabababab"
- ]
- },
- "stacktrace_snippet": {
- "type": "string",
- "description": "The stack trace recorded during fuzzing resulting the crash.",
- "examples": [
- "func_a+0xabcd\nfunc_b+0xabcc"
- ]
- },
- "crash_state": {
- "type": "string",
- "description": "Minimised and normalized crash stack-trace (called crash_state).",
- "examples": [
- "func_a+0xa\nfunc_b+0xb\nfunc_c+0xc"
- ]
- },
- "crash_type": {
- "type": "string",
- "description": "Type of the crash.",
- "examples": [
- "Heap-Buffer-overflow",
- "Division-by-zero"
- ]
- }
- }
- }
- }
- }
- },
- "remediations": {
- "type": "array",
- "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
- "items": {
- "type": "object",
- "required": [
- "fixes",
- "summary",
- "diff"
- ],
- "properties": {
- "fixes": {
- "type": "array",
- "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
- "items": {
- "type": "object",
- "required": [
- "cve"
- ],
- "properties": {
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- }
- }
- }
- },
- "summary": {
- "type": "string",
- "minLength": 1,
- "description": "An overview of how the vulnerabilities were fixed."
- },
- "diff": {
- "type": "string",
- "minLength": 1,
- "description": "A base64-encoded remediation code diff, compatible with git apply."
- }
- }
- }
- }
- }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.6/dependency-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.6/dependency-scanning-report-format.json
deleted file mode 100644
index 80d6fc9c7d2..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.6/dependency-scanning-report-format.json
+++ /dev/null
@@ -1,968 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Report format for GitLab Dependency Scanning",
- "description": "This schema provides the the report format for Dependency Scanning analyzers (https://docs.gitlab.com/ee/user/application_security/dependency_scanning).",
- "definitions": {
- "detail_type": {
- "oneOf": [
- {
- "$ref": "#/definitions/named_list"
- },
- {
- "$ref": "#/definitions/list"
- },
- {
- "$ref": "#/definitions/table"
- },
- {
- "$ref": "#/definitions/text"
- },
- {
- "$ref": "#/definitions/url"
- },
- {
- "$ref": "#/definitions/code"
- },
- {
- "$ref": "#/definitions/value"
- },
- {
- "$ref": "#/definitions/diff"
- },
- {
- "$ref": "#/definitions/markdown"
- },
- {
- "$ref": "#/definitions/commit"
- },
- {
- "$ref": "#/definitions/file_location"
- },
- {
- "$ref": "#/definitions/module_location"
- }
- ]
- },
- "text_value": {
- "type": "string"
- },
- "named_field": {
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "$ref": "#/definitions/text_value",
- "minLength": 1
- },
- "description": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "named_list": {
- "type": "object",
- "description": "An object with named and typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "named-list"
- },
- "items": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- {
- "$ref": "#/definitions/named_field"
- },
- {
- "$ref": "#/definitions/detail_type"
- }
- ]
- }
- }
- }
- }
- },
- "list": {
- "type": "object",
- "description": "A list of typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- },
- "table": {
- "type": "object",
- "description": "A table of typed fields",
- "required": [
- "type",
- "rows"
- ],
- "properties": {
- "type": {
- "const": "table"
- },
- "header": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- },
- "rows": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- }
- },
- "text": {
- "type": "object",
- "description": "Raw text",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "text"
- },
- "value": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "url": {
- "type": "object",
- "description": "A single URL",
- "required": [
- "type",
- "href"
- ],
- "properties": {
- "type": {
- "const": "url"
- },
- "text": {
- "$ref": "#/definitions/text_value"
- },
- "href": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "http://mysite.com"
- ]
- }
- }
- },
- "code": {
- "type": "object",
- "description": "A codeblock",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "code"
- },
- "value": {
- "type": "string"
- },
- "lang": {
- "type": "string",
- "description": "A programming language"
- }
- }
- },
- "value": {
- "type": "object",
- "description": "A field that can store a range of types of value",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "value"
- },
- "value": {
- "type": [
- "number",
- "string",
- "boolean"
- ]
- }
- }
- },
- "diff": {
- "type": "object",
- "description": "A diff",
- "required": [
- "type",
- "before",
- "after"
- ],
- "properties": {
- "type": {
- "const": "diff"
- },
- "before": {
- "type": "string"
- },
- "after": {
- "type": "string"
- }
- }
- },
- "markdown": {
- "type": "object",
- "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "markdown"
- },
- "value": {
- "$ref": "#/definitions/text_value",
- "examples": [
- "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
- ]
- }
- }
- },
- "commit": {
- "type": "object",
- "description": "A commit/tag/branch within the GitLab project",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "commit"
- },
- "value": {
- "type": "string",
- "description": "The commit SHA",
- "minLength": 1
- }
- }
- },
- "file_location": {
- "type": "object",
- "description": "A location within a file in the project",
- "required": [
- "type",
- "file_name",
- "line_start"
- ],
- "properties": {
- "type": {
- "const": "file-location"
- },
- "file_name": {
- "type": "string",
- "minLength": 1
- },
- "line_start": {
- "type": "integer"
- },
- "line_end": {
- "type": "integer"
- }
- }
- },
- "module_location": {
- "type": "object",
- "description": "A location within a binary module of the form module+relative_offset",
- "required": [
- "type",
- "module_name",
- "offset"
- ],
- "properties": {
- "type": {
- "const": "module-location"
- },
- "module_name": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "compiled_binary"
- ]
- },
- "offset": {
- "type": "integer",
- "examples": [
- 100
- ]
- }
- }
- }
- },
- "self": {
- "version": "14.0.6"
- },
- "required": [
- "dependency_files",
- "version",
- "vulnerabilities"
- ],
- "additionalProperties": true,
- "properties": {
- "scan": {
- "type": "object",
- "required": [
- "end_time",
- "scanner",
- "start_time",
- "status",
- "type"
- ],
- "properties": {
- "end_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-01-28T03:26:02"
- ]
- },
- "messages": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Communication intended for the initiator of a scan.",
- "required": [
- "level",
- "value"
- ],
- "properties": {
- "level": {
- "type": "string",
- "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
- "enum": [
- "info",
- "warn",
- "fatal"
- ],
- "examples": [
- "info"
- ]
- },
- "value": {
- "type": "string",
- "description": "The message to communicate.",
- "minLength": 1,
- "examples": [
- "Permission denied, scanning aborted"
- ]
- }
- }
- }
- },
- "analyzer": {
- "type": "object",
- "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the analyzer.",
- "minLength": 1,
- "examples": [
- "gitlab-dast"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the analyzer, not required to be unique.",
- "minLength": 1,
- "examples": [
- "GitLab DAST"
- ]
- },
- "url": {
- "type": "string",
- "format": "uri",
- "pattern": "^https?://.+",
- "description": "A link to more information about the analyzer.",
- "examples": [
- "https://docs.gitlab.com/ee/user/application_security/dast"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the analyzer.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- },
- "version": {
- "type": "string",
- "description": "The version of the analyzer.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- }
- }
- },
- "scanner": {
- "type": "object",
- "description": "Object defining the scanner used to perform the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the scanner.",
- "minLength": 1,
- "examples": [
- "my-sast-scanner"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the scanner, not required to be unique.",
- "minLength": 1,
- "examples": [
- "My SAST Scanner"
- ]
- },
- "url": {
- "type": "string",
- "description": "A link to more information about the scanner.",
- "examples": [
- "https://scanner.url"
- ]
- },
- "version": {
- "type": "string",
- "description": "The version of the scanner.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the scanner.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- }
- }
- },
- "start_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-02-14T16:01:59"
- ]
- },
- "status": {
- "type": "string",
- "description": "Result of the scan.",
- "enum": [
- "success",
- "failure"
- ]
- },
- "type": {
- "type": "string",
- "description": "Type of the scan.",
- "enum": [
- "dependency_scanning"
- ]
- }
- }
- },
- "schema": {
- "type": "string",
- "description": "URI pointing to the validating security report schema.",
- "format": "uri"
- },
- "version": {
- "type": "string",
- "description": "The version of the schema to which the JSON report conforms.",
- "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
- },
- "vulnerabilities": {
- "type": "array",
- "description": "Array of vulnerability objects.",
- "items": {
- "type": "object",
- "description": "Describes the vulnerability using GitLab Flavored Markdown",
- "required": [
- "category",
- "cve",
- "identifiers",
- "location",
- "scanner"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
- "examples": [
- "642735a5-1425-428d-8d4e-3c854885a3c9"
- ]
- },
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
- "name": {
- "type": "string",
- "description": "The name of the vulnerability. This must not include the finding's specific information."
- },
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
- "description": {
- "type": "string",
- "description": "A long text section describing the vulnerability more fully."
- },
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
- "severity": {
- "type": "string",
- "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Info",
- "Unknown",
- "Low",
- "Medium",
- "High",
- "Critical"
- ]
- },
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
- "solution": {
- "type": "string",
- "description": "Explanation of how to fix the vulnerability."
- },
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
- "identifiers": {
- "type": "array",
- "minItems": 1,
- "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
- "items": {
- "type": "object",
- "required": [
- "type",
- "name",
- "value"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
- "minLength": 1
- },
- "name": {
- "type": "string",
- "description": "Human-readable name of the identifier.",
- "minLength": 1
- },
- "url": {
- "type": "string",
- "description": "URL of the identifier's documentation.",
- "format": "uri"
- },
- "value": {
- "type": "string",
- "description": "Value of the identifier, for matching purpose.",
- "minLength": 1
- }
- }
- }
- },
- "links": {
- "type": "array",
- "description": "An array of references to external documentation or articles that describe the vulnerability.",
- "items": {
- "type": "object",
- "required": [
- "url"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the vulnerability details link."
- },
- "url": {
- "type": "string",
- "description": "URL of the vulnerability details document.",
- "format": "uri"
- }
- }
- }
- },
- "details": {
- "$ref": "#/definitions/named_list/properties/items"
- },
- "tracking": {
- "description": "Describes how this vulnerability should be tracked as the project changes.",
- "oneOf": [
- {
- "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
- "required": [
- "items"
- ],
- "properties": {
- "type": {
- "const": "source"
- },
- "items": {
- "type": "array",
- "items": {
- "description": "An item that should be tracked using source-specific tracking methods.",
- "type": "object",
- "required": [
- "signatures"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the file that includes the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the file that includes the vulnerability."
- },
- "signatures": {
- "type": "array",
- "description": "An array of calculated tracking signatures for this tracking item.",
- "minItems": 1,
- "items": {
- "description": "A calculated tracking signature value and metadata.",
- "required": [
- "algorithm",
- "value"
- ],
- "properties": {
- "algorithm": {
- "type": "string",
- "description": "The algorithm used to generate the signature."
- },
- "value": {
- "type": "string",
- "description": "The result of this signature algorithm."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "Each tracking type must declare its own type."
- }
- }
- },
- "flags": {
- "description": "Flags that can be attached to vulnerabilities.",
- "type": "array",
- "items": {
- "type": "object",
- "description": "Informational flags identified and assigned to a vulnerability.",
- "required": [
- "type",
- "origin",
- "description"
- ],
- "properties": {
- "type": {
- "type": "string",
- "minLength": 1,
- "description": "Result of the scan.",
- "enum": [
- "flagged-as-likely-false-positive"
- ]
- },
- "origin": {
- "minLength": 1,
- "description": "Tool that issued the flag.",
- "type": "string"
- },
- "description": {
- "minLength": 1,
- "description": "What the flag is about.",
- "type": "string"
- }
- }
- }
- },
- "location": {
- "type": "object",
- "description": "Identifies the vulnerability's location.",
- "required": [
- "file",
- "dependency"
- ],
- "properties": {
- "file": {
- "type": "string",
- "minLength": 1,
- "description": "Path to the manifest or lock file where the dependency is declared (such as yarn.lock)."
- },
- "dependency": {
- "type": "object",
- "description": "Describes the dependency of a project where the vulnerability is located.",
- "properties": {
- "package": {
- "type": "object",
- "description": "Provides information on the package where the vulnerability is located.",
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the package where the vulnerability is located."
- }
- }
- },
- "version": {
- "type": "string",
- "description": "Version of the vulnerable package."
- },
- "iid": {
- "description": "ID that identifies the dependency in the scope of a dependency file.",
- "type": "number"
- },
- "direct": {
- "type": "boolean",
- "description": "Tells whether this is a direct, top-level dependency of the scanned project."
- },
- "dependency_path": {
- "type": "array",
- "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.",
- "items": {
- "type": "object",
- "required": [
- "iid"
- ],
- "properties": {
- "iid": {
- "type": "number",
- "description": "ID that is unique in the scope of a parent object, and specific to the resource type."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- }
- },
- "remediations": {
- "type": "array",
- "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
- "items": {
- "type": "object",
- "required": [
- "fixes",
- "summary",
- "diff"
- ],
- "properties": {
- "fixes": {
- "type": "array",
- "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
- "items": {
- "type": "object",
- "required": [
- "cve"
- ],
- "properties": {
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- }
- }
- }
- },
- "summary": {
- "type": "string",
- "minLength": 1,
- "description": "An overview of how the vulnerabilities were fixed."
- },
- "diff": {
- "type": "string",
- "minLength": 1,
- "description": "A base64-encoded remediation code diff, compatible with git apply."
- }
- }
- }
- },
- "dependency_files": {
- "type": "array",
- "description": "List of dependency files identified in the project.",
- "items": {
- "type": "object",
- "required": [
- "path",
- "package_manager",
- "dependencies"
- ],
- "properties": {
- "path": {
- "type": "string",
- "minLength": 1
- },
- "package_manager": {
- "type": "string",
- "minLength": 1
- },
- "dependencies": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Describes the dependency of a project where the vulnerability is located.",
- "properties": {
- "package": {
- "type": "object",
- "description": "Provides information on the package where the vulnerability is located.",
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the package where the vulnerability is located."
- }
- }
- },
- "version": {
- "type": "string",
- "description": "Version of the vulnerable package."
- },
- "iid": {
- "description": "ID that identifies the dependency in the scope of a dependency file.",
- "type": "number"
- },
- "direct": {
- "type": "boolean",
- "description": "Tells whether this is a direct, top-level dependency of the scanned project."
- },
- "dependency_path": {
- "type": "array",
- "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.",
- "items": {
- "type": "object",
- "required": [
- "iid"
- ],
- "properties": {
- "iid": {
- "type": "number",
- "description": "ID that is unique in the scope of a parent object, and specific to the resource type."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- }
- }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.6/sast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.6/sast-report-format.json
deleted file mode 100644
index b87182bb237..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.6/sast-report-format.json
+++ /dev/null
@@ -1,869 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Report format for GitLab SAST",
- "description": "This schema provides the report format for Static Application Security Testing analyzers (https://docs.gitlab.com/ee/user/application_security/sast).",
- "definitions": {
- "detail_type": {
- "oneOf": [
- {
- "$ref": "#/definitions/named_list"
- },
- {
- "$ref": "#/definitions/list"
- },
- {
- "$ref": "#/definitions/table"
- },
- {
- "$ref": "#/definitions/text"
- },
- {
- "$ref": "#/definitions/url"
- },
- {
- "$ref": "#/definitions/code"
- },
- {
- "$ref": "#/definitions/value"
- },
- {
- "$ref": "#/definitions/diff"
- },
- {
- "$ref": "#/definitions/markdown"
- },
- {
- "$ref": "#/definitions/commit"
- },
- {
- "$ref": "#/definitions/file_location"
- },
- {
- "$ref": "#/definitions/module_location"
- }
- ]
- },
- "text_value": {
- "type": "string"
- },
- "named_field": {
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "$ref": "#/definitions/text_value",
- "minLength": 1
- },
- "description": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "named_list": {
- "type": "object",
- "description": "An object with named and typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "named-list"
- },
- "items": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- {
- "$ref": "#/definitions/named_field"
- },
- {
- "$ref": "#/definitions/detail_type"
- }
- ]
- }
- }
- }
- }
- },
- "list": {
- "type": "object",
- "description": "A list of typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- },
- "table": {
- "type": "object",
- "description": "A table of typed fields",
- "required": [
- "type",
- "rows"
- ],
- "properties": {
- "type": {
- "const": "table"
- },
- "header": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- },
- "rows": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- }
- },
- "text": {
- "type": "object",
- "description": "Raw text",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "text"
- },
- "value": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "url": {
- "type": "object",
- "description": "A single URL",
- "required": [
- "type",
- "href"
- ],
- "properties": {
- "type": {
- "const": "url"
- },
- "text": {
- "$ref": "#/definitions/text_value"
- },
- "href": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "http://mysite.com"
- ]
- }
- }
- },
- "code": {
- "type": "object",
- "description": "A codeblock",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "code"
- },
- "value": {
- "type": "string"
- },
- "lang": {
- "type": "string",
- "description": "A programming language"
- }
- }
- },
- "value": {
- "type": "object",
- "description": "A field that can store a range of types of value",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "value"
- },
- "value": {
- "type": [
- "number",
- "string",
- "boolean"
- ]
- }
- }
- },
- "diff": {
- "type": "object",
- "description": "A diff",
- "required": [
- "type",
- "before",
- "after"
- ],
- "properties": {
- "type": {
- "const": "diff"
- },
- "before": {
- "type": "string"
- },
- "after": {
- "type": "string"
- }
- }
- },
- "markdown": {
- "type": "object",
- "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "markdown"
- },
- "value": {
- "$ref": "#/definitions/text_value",
- "examples": [
- "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
- ]
- }
- }
- },
- "commit": {
- "type": "object",
- "description": "A commit/tag/branch within the GitLab project",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "commit"
- },
- "value": {
- "type": "string",
- "description": "The commit SHA",
- "minLength": 1
- }
- }
- },
- "file_location": {
- "type": "object",
- "description": "A location within a file in the project",
- "required": [
- "type",
- "file_name",
- "line_start"
- ],
- "properties": {
- "type": {
- "const": "file-location"
- },
- "file_name": {
- "type": "string",
- "minLength": 1
- },
- "line_start": {
- "type": "integer"
- },
- "line_end": {
- "type": "integer"
- }
- }
- },
- "module_location": {
- "type": "object",
- "description": "A location within a binary module of the form module+relative_offset",
- "required": [
- "type",
- "module_name",
- "offset"
- ],
- "properties": {
- "type": {
- "const": "module-location"
- },
- "module_name": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "compiled_binary"
- ]
- },
- "offset": {
- "type": "integer",
- "examples": [
- 100
- ]
- }
- }
- }
- },
- "self": {
- "version": "14.0.6"
- },
- "required": [
- "version",
- "vulnerabilities"
- ],
- "additionalProperties": true,
- "properties": {
- "scan": {
- "type": "object",
- "required": [
- "end_time",
- "scanner",
- "start_time",
- "status",
- "type"
- ],
- "properties": {
- "end_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-01-28T03:26:02"
- ]
- },
- "messages": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Communication intended for the initiator of a scan.",
- "required": [
- "level",
- "value"
- ],
- "properties": {
- "level": {
- "type": "string",
- "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
- "enum": [
- "info",
- "warn",
- "fatal"
- ],
- "examples": [
- "info"
- ]
- },
- "value": {
- "type": "string",
- "description": "The message to communicate.",
- "minLength": 1,
- "examples": [
- "Permission denied, scanning aborted"
- ]
- }
- }
- }
- },
- "analyzer": {
- "type": "object",
- "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the analyzer.",
- "minLength": 1,
- "examples": [
- "gitlab-dast"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the analyzer, not required to be unique.",
- "minLength": 1,
- "examples": [
- "GitLab DAST"
- ]
- },
- "url": {
- "type": "string",
- "format": "uri",
- "pattern": "^https?://.+",
- "description": "A link to more information about the analyzer.",
- "examples": [
- "https://docs.gitlab.com/ee/user/application_security/dast"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the analyzer.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- },
- "version": {
- "type": "string",
- "description": "The version of the analyzer.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- }
- }
- },
- "scanner": {
- "type": "object",
- "description": "Object defining the scanner used to perform the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the scanner.",
- "minLength": 1,
- "examples": [
- "my-sast-scanner"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the scanner, not required to be unique.",
- "minLength": 1,
- "examples": [
- "My SAST Scanner"
- ]
- },
- "url": {
- "type": "string",
- "description": "A link to more information about the scanner.",
- "examples": [
- "https://scanner.url"
- ]
- },
- "version": {
- "type": "string",
- "description": "The version of the scanner.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the scanner.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- }
- }
- },
- "start_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-02-14T16:01:59"
- ]
- },
- "status": {
- "type": "string",
- "description": "Result of the scan.",
- "enum": [
- "success",
- "failure"
- ]
- },
- "type": {
- "type": "string",
- "description": "Type of the scan.",
- "enum": [
- "sast"
- ]
- }
- }
- },
- "schema": {
- "type": "string",
- "description": "URI pointing to the validating security report schema.",
- "format": "uri"
- },
- "version": {
- "type": "string",
- "description": "The version of the schema to which the JSON report conforms.",
- "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
- },
- "vulnerabilities": {
- "type": "array",
- "description": "Array of vulnerability objects.",
- "items": {
- "type": "object",
- "description": "Describes the vulnerability using GitLab Flavored Markdown",
- "required": [
- "category",
- "cve",
- "identifiers",
- "location",
- "scanner"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
- "examples": [
- "642735a5-1425-428d-8d4e-3c854885a3c9"
- ]
- },
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
- "name": {
- "type": "string",
- "description": "The name of the vulnerability. This must not include the finding's specific information."
- },
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
- "description": {
- "type": "string",
- "description": "A long text section describing the vulnerability more fully."
- },
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
- "severity": {
- "type": "string",
- "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Info",
- "Unknown",
- "Low",
- "Medium",
- "High",
- "Critical"
- ]
- },
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
- "solution": {
- "type": "string",
- "description": "Explanation of how to fix the vulnerability."
- },
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
- "identifiers": {
- "type": "array",
- "minItems": 1,
- "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
- "items": {
- "type": "object",
- "required": [
- "type",
- "name",
- "value"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
- "minLength": 1
- },
- "name": {
- "type": "string",
- "description": "Human-readable name of the identifier.",
- "minLength": 1
- },
- "url": {
- "type": "string",
- "description": "URL of the identifier's documentation.",
- "format": "uri"
- },
- "value": {
- "type": "string",
- "description": "Value of the identifier, for matching purpose.",
- "minLength": 1
- }
- }
- }
- },
- "links": {
- "type": "array",
- "description": "An array of references to external documentation or articles that describe the vulnerability.",
- "items": {
- "type": "object",
- "required": [
- "url"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the vulnerability details link."
- },
- "url": {
- "type": "string",
- "description": "URL of the vulnerability details document.",
- "format": "uri"
- }
- }
- }
- },
- "details": {
- "$ref": "#/definitions/named_list/properties/items"
- },
- "tracking": {
- "description": "Describes how this vulnerability should be tracked as the project changes.",
- "oneOf": [
- {
- "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
- "required": [
- "items"
- ],
- "properties": {
- "type": {
- "const": "source"
- },
- "items": {
- "type": "array",
- "items": {
- "description": "An item that should be tracked using source-specific tracking methods.",
- "type": "object",
- "required": [
- "signatures"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the file that includes the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the file that includes the vulnerability."
- },
- "signatures": {
- "type": "array",
- "description": "An array of calculated tracking signatures for this tracking item.",
- "minItems": 1,
- "items": {
- "description": "A calculated tracking signature value and metadata.",
- "required": [
- "algorithm",
- "value"
- ],
- "properties": {
- "algorithm": {
- "type": "string",
- "description": "The algorithm used to generate the signature."
- },
- "value": {
- "type": "string",
- "description": "The result of this signature algorithm."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "Each tracking type must declare its own type."
- }
- }
- },
- "flags": {
- "description": "Flags that can be attached to vulnerabilities.",
- "type": "array",
- "items": {
- "type": "object",
- "description": "Informational flags identified and assigned to a vulnerability.",
- "required": [
- "type",
- "origin",
- "description"
- ],
- "properties": {
- "type": {
- "type": "string",
- "minLength": 1,
- "description": "Result of the scan.",
- "enum": [
- "flagged-as-likely-false-positive"
- ]
- },
- "origin": {
- "minLength": 1,
- "description": "Tool that issued the flag.",
- "type": "string"
- },
- "description": {
- "minLength": 1,
- "description": "What the flag is about.",
- "type": "string"
- }
- }
- }
- },
- "location": {
- "type": "object",
- "description": "Identifies the vulnerability's location.",
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the code affected by the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the code affected by the vulnerability."
- },
- "class": {
- "type": "string",
- "description": "Provides the name of the class where the vulnerability is located."
- },
- "method": {
- "type": "string",
- "description": "Provides the name of the method where the vulnerability is located."
- }
- }
- },
- "raw_source_code_extract": {
- "type": "string",
- "description": "Provides an unsanitized excerpt of the affected source code."
- }
- }
- }
- },
- "remediations": {
- "type": "array",
- "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
- "items": {
- "type": "object",
- "required": [
- "fixes",
- "summary",
- "diff"
- ],
- "properties": {
- "fixes": {
- "type": "array",
- "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
- "items": {
- "type": "object",
- "required": [
- "cve"
- ],
- "properties": {
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- }
- }
- }
- },
- "summary": {
- "type": "string",
- "minLength": 1,
- "description": "An overview of how the vulnerabilities were fixed."
- },
- "diff": {
- "type": "string",
- "minLength": 1,
- "description": "A base64-encoded remediation code diff, compatible with git apply."
- }
- }
- }
- }
- }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.6/secret-detection-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.6/secret-detection-report-format.json
deleted file mode 100644
index 191d94aad5f..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.6/secret-detection-report-format.json
+++ /dev/null
@@ -1,892 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Report format for GitLab Secret Detection",
- "description": "This schema provides the the report format for the Secret Detection analyzer (https://docs.gitlab.com/ee/user/application_security/secret_detection)",
- "definitions": {
- "detail_type": {
- "oneOf": [
- {
- "$ref": "#/definitions/named_list"
- },
- {
- "$ref": "#/definitions/list"
- },
- {
- "$ref": "#/definitions/table"
- },
- {
- "$ref": "#/definitions/text"
- },
- {
- "$ref": "#/definitions/url"
- },
- {
- "$ref": "#/definitions/code"
- },
- {
- "$ref": "#/definitions/value"
- },
- {
- "$ref": "#/definitions/diff"
- },
- {
- "$ref": "#/definitions/markdown"
- },
- {
- "$ref": "#/definitions/commit"
- },
- {
- "$ref": "#/definitions/file_location"
- },
- {
- "$ref": "#/definitions/module_location"
- }
- ]
- },
- "text_value": {
- "type": "string"
- },
- "named_field": {
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "$ref": "#/definitions/text_value",
- "minLength": 1
- },
- "description": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "named_list": {
- "type": "object",
- "description": "An object with named and typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "named-list"
- },
- "items": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- {
- "$ref": "#/definitions/named_field"
- },
- {
- "$ref": "#/definitions/detail_type"
- }
- ]
- }
- }
- }
- }
- },
- "list": {
- "type": "object",
- "description": "A list of typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- },
- "table": {
- "type": "object",
- "description": "A table of typed fields",
- "required": [
- "type",
- "rows"
- ],
- "properties": {
- "type": {
- "const": "table"
- },
- "header": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- },
- "rows": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- }
- },
- "text": {
- "type": "object",
- "description": "Raw text",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "text"
- },
- "value": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "url": {
- "type": "object",
- "description": "A single URL",
- "required": [
- "type",
- "href"
- ],
- "properties": {
- "type": {
- "const": "url"
- },
- "text": {
- "$ref": "#/definitions/text_value"
- },
- "href": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "http://mysite.com"
- ]
- }
- }
- },
- "code": {
- "type": "object",
- "description": "A codeblock",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "code"
- },
- "value": {
- "type": "string"
- },
- "lang": {
- "type": "string",
- "description": "A programming language"
- }
- }
- },
- "value": {
- "type": "object",
- "description": "A field that can store a range of types of value",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "value"
- },
- "value": {
- "type": [
- "number",
- "string",
- "boolean"
- ]
- }
- }
- },
- "diff": {
- "type": "object",
- "description": "A diff",
- "required": [
- "type",
- "before",
- "after"
- ],
- "properties": {
- "type": {
- "const": "diff"
- },
- "before": {
- "type": "string"
- },
- "after": {
- "type": "string"
- }
- }
- },
- "markdown": {
- "type": "object",
- "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "markdown"
- },
- "value": {
- "$ref": "#/definitions/text_value",
- "examples": [
- "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
- ]
- }
- }
- },
- "commit": {
- "type": "object",
- "description": "A commit/tag/branch within the GitLab project",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "commit"
- },
- "value": {
- "type": "string",
- "description": "The commit SHA",
- "minLength": 1
- }
- }
- },
- "file_location": {
- "type": "object",
- "description": "A location within a file in the project",
- "required": [
- "type",
- "file_name",
- "line_start"
- ],
- "properties": {
- "type": {
- "const": "file-location"
- },
- "file_name": {
- "type": "string",
- "minLength": 1
- },
- "line_start": {
- "type": "integer"
- },
- "line_end": {
- "type": "integer"
- }
- }
- },
- "module_location": {
- "type": "object",
- "description": "A location within a binary module of the form module+relative_offset",
- "required": [
- "type",
- "module_name",
- "offset"
- ],
- "properties": {
- "type": {
- "const": "module-location"
- },
- "module_name": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "compiled_binary"
- ]
- },
- "offset": {
- "type": "integer",
- "examples": [
- 100
- ]
- }
- }
- }
- },
- "self": {
- "version": "14.0.6"
- },
- "required": [
- "version",
- "vulnerabilities"
- ],
- "additionalProperties": true,
- "properties": {
- "scan": {
- "type": "object",
- "required": [
- "end_time",
- "scanner",
- "start_time",
- "status",
- "type"
- ],
- "properties": {
- "end_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-01-28T03:26:02"
- ]
- },
- "messages": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Communication intended for the initiator of a scan.",
- "required": [
- "level",
- "value"
- ],
- "properties": {
- "level": {
- "type": "string",
- "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
- "enum": [
- "info",
- "warn",
- "fatal"
- ],
- "examples": [
- "info"
- ]
- },
- "value": {
- "type": "string",
- "description": "The message to communicate.",
- "minLength": 1,
- "examples": [
- "Permission denied, scanning aborted"
- ]
- }
- }
- }
- },
- "analyzer": {
- "type": "object",
- "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the analyzer.",
- "minLength": 1,
- "examples": [
- "gitlab-dast"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the analyzer, not required to be unique.",
- "minLength": 1,
- "examples": [
- "GitLab DAST"
- ]
- },
- "url": {
- "type": "string",
- "format": "uri",
- "pattern": "^https?://.+",
- "description": "A link to more information about the analyzer.",
- "examples": [
- "https://docs.gitlab.com/ee/user/application_security/dast"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the analyzer.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- },
- "version": {
- "type": "string",
- "description": "The version of the analyzer.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- }
- }
- },
- "scanner": {
- "type": "object",
- "description": "Object defining the scanner used to perform the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the scanner.",
- "minLength": 1,
- "examples": [
- "my-sast-scanner"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the scanner, not required to be unique.",
- "minLength": 1,
- "examples": [
- "My SAST Scanner"
- ]
- },
- "url": {
- "type": "string",
- "description": "A link to more information about the scanner.",
- "examples": [
- "https://scanner.url"
- ]
- },
- "version": {
- "type": "string",
- "description": "The version of the scanner.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the scanner.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- }
- }
- },
- "start_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-02-14T16:01:59"
- ]
- },
- "status": {
- "type": "string",
- "description": "Result of the scan.",
- "enum": [
- "success",
- "failure"
- ]
- },
- "type": {
- "type": "string",
- "description": "Type of the scan.",
- "enum": [
- "secret_detection"
- ]
- }
- }
- },
- "schema": {
- "type": "string",
- "description": "URI pointing to the validating security report schema.",
- "format": "uri"
- },
- "version": {
- "type": "string",
- "description": "The version of the schema to which the JSON report conforms.",
- "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
- },
- "vulnerabilities": {
- "type": "array",
- "description": "Array of vulnerability objects.",
- "items": {
- "type": "object",
- "description": "Describes the vulnerability using GitLab Flavored Markdown",
- "required": [
- "category",
- "cve",
- "identifiers",
- "location",
- "scanner"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
- "examples": [
- "642735a5-1425-428d-8d4e-3c854885a3c9"
- ]
- },
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
- "name": {
- "type": "string",
- "description": "The name of the vulnerability. This must not include the finding's specific information."
- },
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
- "description": {
- "type": "string",
- "description": "A long text section describing the vulnerability more fully."
- },
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
- "severity": {
- "type": "string",
- "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Info",
- "Unknown",
- "Low",
- "Medium",
- "High",
- "Critical"
- ]
- },
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
- "solution": {
- "type": "string",
- "description": "Explanation of how to fix the vulnerability."
- },
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
- "identifiers": {
- "type": "array",
- "minItems": 1,
- "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
- "items": {
- "type": "object",
- "required": [
- "type",
- "name",
- "value"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
- "minLength": 1
- },
- "name": {
- "type": "string",
- "description": "Human-readable name of the identifier.",
- "minLength": 1
- },
- "url": {
- "type": "string",
- "description": "URL of the identifier's documentation.",
- "format": "uri"
- },
- "value": {
- "type": "string",
- "description": "Value of the identifier, for matching purpose.",
- "minLength": 1
- }
- }
- }
- },
- "links": {
- "type": "array",
- "description": "An array of references to external documentation or articles that describe the vulnerability.",
- "items": {
- "type": "object",
- "required": [
- "url"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the vulnerability details link."
- },
- "url": {
- "type": "string",
- "description": "URL of the vulnerability details document.",
- "format": "uri"
- }
- }
- }
- },
- "details": {
- "$ref": "#/definitions/named_list/properties/items"
- },
- "tracking": {
- "description": "Describes how this vulnerability should be tracked as the project changes.",
- "oneOf": [
- {
- "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
- "required": [
- "items"
- ],
- "properties": {
- "type": {
- "const": "source"
- },
- "items": {
- "type": "array",
- "items": {
- "description": "An item that should be tracked using source-specific tracking methods.",
- "type": "object",
- "required": [
- "signatures"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the file that includes the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the file that includes the vulnerability."
- },
- "signatures": {
- "type": "array",
- "description": "An array of calculated tracking signatures for this tracking item.",
- "minItems": 1,
- "items": {
- "description": "A calculated tracking signature value and metadata.",
- "required": [
- "algorithm",
- "value"
- ],
- "properties": {
- "algorithm": {
- "type": "string",
- "description": "The algorithm used to generate the signature."
- },
- "value": {
- "type": "string",
- "description": "The result of this signature algorithm."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "Each tracking type must declare its own type."
- }
- }
- },
- "flags": {
- "description": "Flags that can be attached to vulnerabilities.",
- "type": "array",
- "items": {
- "type": "object",
- "description": "Informational flags identified and assigned to a vulnerability.",
- "required": [
- "type",
- "origin",
- "description"
- ],
- "properties": {
- "type": {
- "type": "string",
- "minLength": 1,
- "description": "Result of the scan.",
- "enum": [
- "flagged-as-likely-false-positive"
- ]
- },
- "origin": {
- "minLength": 1,
- "description": "Tool that issued the flag.",
- "type": "string"
- },
- "description": {
- "minLength": 1,
- "description": "What the flag is about.",
- "type": "string"
- }
- }
- }
- },
- "location": {
- "required": [
- "commit"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located"
- },
- "commit": {
- "type": "object",
- "description": "Represents the commit in which the vulnerability was detected",
- "required": [
- "sha"
- ],
- "properties": {
- "author": {
- "type": "string"
- },
- "date": {
- "type": "string"
- },
- "message": {
- "type": "string"
- },
- "sha": {
- "type": "string",
- "minLength": 1
- }
- }
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the code affected by the vulnerability"
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the code affected by the vulnerability"
- },
- "class": {
- "type": "string",
- "description": "Provides the name of the class where the vulnerability is located"
- },
- "method": {
- "type": "string",
- "description": "Provides the name of the method where the vulnerability is located"
- }
- }
- },
- "raw_source_code_extract": {
- "type": "string",
- "description": "Provides an unsanitized excerpt of the affected source code."
- }
- }
- }
- },
- "remediations": {
- "type": "array",
- "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
- "items": {
- "type": "object",
- "required": [
- "fixes",
- "summary",
- "diff"
- ],
- "properties": {
- "fixes": {
- "type": "array",
- "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
- "items": {
- "type": "object",
- "required": [
- "cve"
- ],
- "properties": {
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- }
- }
- }
- },
- "summary": {
- "type": "string",
- "minLength": 1,
- "description": "An overview of how the vulnerabilities were fixed."
- },
- "diff": {
- "type": "string",
- "minLength": 1,
- "description": "A base64-encoded remediation code diff, compatible with git apply."
- }
- }
- }
- }
- }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.0/cluster-image-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.0/cluster-image-scanning-report-format.json
deleted file mode 100644
index 3f78ff0354f..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.0/cluster-image-scanning-report-format.json
+++ /dev/null
@@ -1,977 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Report format for GitLab Cluster Image Scanning",
- "description": "This schema provides the the report format for Cluster Image Scanning (https://docs.gitlab.com/ee/user/application_security/cluster_image_scanning/).",
- "definitions": {
- "detail_type": {
- "oneOf": [
- {
- "$ref": "#/definitions/named_list"
- },
- {
- "$ref": "#/definitions/list"
- },
- {
- "$ref": "#/definitions/table"
- },
- {
- "$ref": "#/definitions/text"
- },
- {
- "$ref": "#/definitions/url"
- },
- {
- "$ref": "#/definitions/code"
- },
- {
- "$ref": "#/definitions/value"
- },
- {
- "$ref": "#/definitions/diff"
- },
- {
- "$ref": "#/definitions/markdown"
- },
- {
- "$ref": "#/definitions/commit"
- },
- {
- "$ref": "#/definitions/file_location"
- },
- {
- "$ref": "#/definitions/module_location"
- }
- ]
- },
- "text_value": {
- "type": "string"
- },
- "named_field": {
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "$ref": "#/definitions/text_value",
- "minLength": 1
- },
- "description": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "named_list": {
- "type": "object",
- "description": "An object with named and typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "named-list"
- },
- "items": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- {
- "$ref": "#/definitions/named_field"
- },
- {
- "$ref": "#/definitions/detail_type"
- }
- ]
- }
- }
- }
- }
- },
- "list": {
- "type": "object",
- "description": "A list of typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- },
- "table": {
- "type": "object",
- "description": "A table of typed fields",
- "required": [
- "type",
- "rows"
- ],
- "properties": {
- "type": {
- "const": "table"
- },
- "header": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- },
- "rows": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- }
- },
- "text": {
- "type": "object",
- "description": "Raw text",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "text"
- },
- "value": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "url": {
- "type": "object",
- "description": "A single URL",
- "required": [
- "type",
- "href"
- ],
- "properties": {
- "type": {
- "const": "url"
- },
- "text": {
- "$ref": "#/definitions/text_value"
- },
- "href": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "http://mysite.com"
- ]
- }
- }
- },
- "code": {
- "type": "object",
- "description": "A codeblock",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "code"
- },
- "value": {
- "type": "string"
- },
- "lang": {
- "type": "string",
- "description": "A programming language"
- }
- }
- },
- "value": {
- "type": "object",
- "description": "A field that can store a range of types of value",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "value"
- },
- "value": {
- "type": [
- "number",
- "string",
- "boolean"
- ]
- }
- }
- },
- "diff": {
- "type": "object",
- "description": "A diff",
- "required": [
- "type",
- "before",
- "after"
- ],
- "properties": {
- "type": {
- "const": "diff"
- },
- "before": {
- "type": "string"
- },
- "after": {
- "type": "string"
- }
- }
- },
- "markdown": {
- "type": "object",
- "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "markdown"
- },
- "value": {
- "$ref": "#/definitions/text_value",
- "examples": [
- "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
- ]
- }
- }
- },
- "commit": {
- "type": "object",
- "description": "A commit/tag/branch within the GitLab project",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "commit"
- },
- "value": {
- "type": "string",
- "description": "The commit SHA",
- "minLength": 1
- }
- }
- },
- "file_location": {
- "type": "object",
- "description": "A location within a file in the project",
- "required": [
- "type",
- "file_name",
- "line_start"
- ],
- "properties": {
- "type": {
- "const": "file-location"
- },
- "file_name": {
- "type": "string",
- "minLength": 1
- },
- "line_start": {
- "type": "integer"
- },
- "line_end": {
- "type": "integer"
- }
- }
- },
- "module_location": {
- "type": "object",
- "description": "A location within a binary module of the form module+relative_offset",
- "required": [
- "type",
- "module_name",
- "offset"
- ],
- "properties": {
- "type": {
- "const": "module-location"
- },
- "module_name": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "compiled_binary"
- ]
- },
- "offset": {
- "type": "integer",
- "examples": [
- 100
- ]
- }
- }
- }
- },
- "self": {
- "version": "14.1.0"
- },
- "required": [
- "version",
- "vulnerabilities"
- ],
- "additionalProperties": true,
- "properties": {
- "scan": {
- "type": "object",
- "required": [
- "end_time",
- "scanner",
- "start_time",
- "status",
- "type"
- ],
- "properties": {
- "end_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-01-28T03:26:02"
- ]
- },
- "messages": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Communication intended for the initiator of a scan.",
- "required": [
- "level",
- "value"
- ],
- "properties": {
- "level": {
- "type": "string",
- "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
- "enum": [
- "info",
- "warn",
- "fatal"
- ],
- "examples": [
- "info"
- ]
- },
- "value": {
- "type": "string",
- "description": "The message to communicate.",
- "minLength": 1,
- "examples": [
- "Permission denied, scanning aborted"
- ]
- }
- }
- }
- },
- "analyzer": {
- "type": "object",
- "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the analyzer.",
- "minLength": 1,
- "examples": [
- "gitlab-dast"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the analyzer, not required to be unique.",
- "minLength": 1,
- "examples": [
- "GitLab DAST"
- ]
- },
- "url": {
- "type": "string",
- "format": "uri",
- "pattern": "^https?://.+",
- "description": "A link to more information about the analyzer.",
- "examples": [
- "https://docs.gitlab.com/ee/user/application_security/dast"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the analyzer.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- },
- "version": {
- "type": "string",
- "description": "The version of the analyzer.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- }
- }
- },
- "scanner": {
- "type": "object",
- "description": "Object defining the scanner used to perform the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the scanner.",
- "minLength": 1,
- "examples": [
- "my-sast-scanner"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the scanner, not required to be unique.",
- "minLength": 1,
- "examples": [
- "My SAST Scanner"
- ]
- },
- "url": {
- "type": "string",
- "description": "A link to more information about the scanner.",
- "examples": [
- "https://scanner.url"
- ]
- },
- "version": {
- "type": "string",
- "description": "The version of the scanner.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the scanner.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- }
- }
- },
- "start_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-02-14T16:01:59"
- ]
- },
- "status": {
- "type": "string",
- "description": "Result of the scan.",
- "enum": [
- "success",
- "failure"
- ]
- },
- "type": {
- "type": "string",
- "description": "Type of the scan.",
- "enum": [
- "cluster_image_scanning"
- ]
- }
- }
- },
- "schema": {
- "type": "string",
- "description": "URI pointing to the validating security report schema.",
- "format": "uri"
- },
- "version": {
- "type": "string",
- "description": "The version of the schema to which the JSON report conforms.",
- "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
- },
- "vulnerabilities": {
- "type": "array",
- "description": "Array of vulnerability objects.",
- "items": {
- "type": "object",
- "description": "Describes the vulnerability using GitLab Flavored Markdown",
- "required": [
- "category",
- "cve",
- "identifiers",
- "location",
- "scanner"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
- "examples": [
- "642735a5-1425-428d-8d4e-3c854885a3c9"
- ]
- },
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
- "name": {
- "type": "string",
- "description": "The name of the vulnerability. This must not include the finding's specific information."
- },
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
- "description": {
- "type": "string",
- "description": "A long text section describing the vulnerability more fully."
- },
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
- "severity": {
- "type": "string",
- "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Info",
- "Unknown",
- "Low",
- "Medium",
- "High",
- "Critical"
- ]
- },
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
- "solution": {
- "type": "string",
- "description": "Explanation of how to fix the vulnerability."
- },
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
- "identifiers": {
- "type": "array",
- "minItems": 1,
- "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
- "items": {
- "type": "object",
- "required": [
- "type",
- "name",
- "value"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
- "minLength": 1
- },
- "name": {
- "type": "string",
- "description": "Human-readable name of the identifier.",
- "minLength": 1
- },
- "url": {
- "type": "string",
- "description": "URL of the identifier's documentation.",
- "format": "uri"
- },
- "value": {
- "type": "string",
- "description": "Value of the identifier, for matching purpose.",
- "minLength": 1
- }
- }
- }
- },
- "links": {
- "type": "array",
- "description": "An array of references to external documentation or articles that describe the vulnerability.",
- "items": {
- "type": "object",
- "required": [
- "url"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the vulnerability details link."
- },
- "url": {
- "type": "string",
- "description": "URL of the vulnerability details document.",
- "format": "uri"
- }
- }
- }
- },
- "details": {
- "$ref": "#/definitions/named_list/properties/items"
- },
- "tracking": {
- "description": "Describes how this vulnerability should be tracked as the project changes.",
- "oneOf": [
- {
- "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
- "required": [
- "items"
- ],
- "properties": {
- "type": {
- "const": "source"
- },
- "items": {
- "type": "array",
- "items": {
- "description": "An item that should be tracked using source-specific tracking methods.",
- "type": "object",
- "required": [
- "signatures"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the file that includes the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the file that includes the vulnerability."
- },
- "signatures": {
- "type": "array",
- "description": "An array of calculated tracking signatures for this tracking item.",
- "minItems": 1,
- "items": {
- "description": "A calculated tracking signature value and metadata.",
- "required": [
- "algorithm",
- "value"
- ],
- "properties": {
- "algorithm": {
- "type": "string",
- "description": "The algorithm used to generate the signature."
- },
- "value": {
- "type": "string",
- "description": "The result of this signature algorithm."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "Each tracking type must declare its own type."
- }
- }
- },
- "flags": {
- "description": "Flags that can be attached to vulnerabilities.",
- "type": "array",
- "items": {
- "type": "object",
- "description": "Informational flags identified and assigned to a vulnerability.",
- "required": [
- "type",
- "origin",
- "description"
- ],
- "properties": {
- "type": {
- "type": "string",
- "minLength": 1,
- "description": "Result of the scan.",
- "enum": [
- "flagged-as-likely-false-positive"
- ]
- },
- "origin": {
- "minLength": 1,
- "description": "Tool that issued the flag.",
- "type": "string"
- },
- "description": {
- "minLength": 1,
- "description": "What the flag is about.",
- "type": "string"
- }
- }
- }
- },
- "location": {
- "type": "object",
- "description": "Identifies the vulnerability's location.",
- "required": [
- "dependency",
- "image",
- "kubernetes_resource"
- ],
- "properties": {
- "dependency": {
- "type": "object",
- "description": "Describes the dependency of a project where the vulnerability is located.",
- "properties": {
- "package": {
- "type": "object",
- "description": "Provides information on the package where the vulnerability is located.",
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the package where the vulnerability is located."
- }
- }
- },
- "version": {
- "type": "string",
- "description": "Version of the vulnerable package."
- },
- "iid": {
- "description": "ID that identifies the dependency in the scope of a dependency file.",
- "type": "number"
- },
- "direct": {
- "type": "boolean",
- "description": "Tells whether this is a direct, top-level dependency of the scanned project."
- },
- "dependency_path": {
- "type": "array",
- "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.",
- "items": {
- "type": "object",
- "required": [
- "iid"
- ],
- "properties": {
- "iid": {
- "type": "number",
- "description": "ID that is unique in the scope of a parent object, and specific to the resource type."
- }
- }
- }
- }
- }
- },
- "operating_system": {
- "type": "string",
- "minLength": 1,
- "maxLength": 255,
- "description": "The operating system that contains the vulnerable package."
- },
- "image": {
- "type": "string",
- "minLength": 1,
- "description": "The analyzed Docker image.",
- "examples": [
- "index.docker.io/library/nginx:1.21"
- ]
- },
- "kubernetes_resource": {
- "type": "object",
- "description": "The specific Kubernetes resource that was scanned.",
- "required": [
- "namespace",
- "kind",
- "name",
- "container_name"
- ],
- "properties": {
- "namespace": {
- "type": "string",
- "minLength": 1,
- "maxLength": 255,
- "description": "The Kubernetes namespace the resource that had its image scanned.",
- "examples": [
- "default",
- "staging",
- "production"
- ]
- },
- "kind": {
- "type": "string",
- "minLength": 1,
- "maxLength": 255,
- "description": "The Kubernetes kind the resource that had its image scanned.",
- "examples": [
- "Deployment",
- "DaemonSet"
- ]
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "maxLength": 255,
- "description": "The name of the resource that had its image scanned.",
- "examples": [
- "nginx-ingress"
- ]
- },
- "container_name": {
- "type": "string",
- "minLength": 1,
- "maxLength": 255,
- "description": "The name of the container that had its image scanned.",
- "examples": [
- "nginx"
- ]
- },
- "agent_id": {
- "type": "string",
- "minLength": 1,
- "maxLength": 255,
- "description": "The GitLab ID of the Kubernetes Agent which performed the scan.",
- "examples": [
- "1234"
- ]
- },
- "cluster_id": {
- "type": "string",
- "minLength": 1,
- "maxLength": 255,
- "description": "The GitLab ID of the Kubernetes cluster when using cluster integration.",
- "examples": [
- "1234"
- ]
- }
- }
- }
- }
- }
- }
- }
- },
- "remediations": {
- "type": "array",
- "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
- "items": {
- "type": "object",
- "required": [
- "fixes",
- "summary",
- "diff"
- ],
- "properties": {
- "fixes": {
- "type": "array",
- "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
- "items": {
- "type": "object",
- "required": [
- "cve"
- ],
- "properties": {
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- }
- }
- }
- },
- "summary": {
- "type": "string",
- "minLength": 1,
- "description": "An overview of how the vulnerabilities were fixed."
- },
- "diff": {
- "type": "string",
- "minLength": 1,
- "description": "A base64-encoded remediation code diff, compatible with git apply."
- }
- }
- }
- }
- }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.0/container-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.0/container-scanning-report-format.json
deleted file mode 100644
index 6e8a1c54fb4..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.0/container-scanning-report-format.json
+++ /dev/null
@@ -1,911 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Report format for GitLab Container Scanning",
- "description": "This schema provides the the report format for Container Scanning (https://docs.gitlab.com/ee/user/application_security/container_scanning).",
- "definitions": {
- "detail_type": {
- "oneOf": [
- {
- "$ref": "#/definitions/named_list"
- },
- {
- "$ref": "#/definitions/list"
- },
- {
- "$ref": "#/definitions/table"
- },
- {
- "$ref": "#/definitions/text"
- },
- {
- "$ref": "#/definitions/url"
- },
- {
- "$ref": "#/definitions/code"
- },
- {
- "$ref": "#/definitions/value"
- },
- {
- "$ref": "#/definitions/diff"
- },
- {
- "$ref": "#/definitions/markdown"
- },
- {
- "$ref": "#/definitions/commit"
- },
- {
- "$ref": "#/definitions/file_location"
- },
- {
- "$ref": "#/definitions/module_location"
- }
- ]
- },
- "text_value": {
- "type": "string"
- },
- "named_field": {
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "$ref": "#/definitions/text_value",
- "minLength": 1
- },
- "description": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "named_list": {
- "type": "object",
- "description": "An object with named and typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "named-list"
- },
- "items": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- {
- "$ref": "#/definitions/named_field"
- },
- {
- "$ref": "#/definitions/detail_type"
- }
- ]
- }
- }
- }
- }
- },
- "list": {
- "type": "object",
- "description": "A list of typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- },
- "table": {
- "type": "object",
- "description": "A table of typed fields",
- "required": [
- "type",
- "rows"
- ],
- "properties": {
- "type": {
- "const": "table"
- },
- "header": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- },
- "rows": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- }
- },
- "text": {
- "type": "object",
- "description": "Raw text",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "text"
- },
- "value": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "url": {
- "type": "object",
- "description": "A single URL",
- "required": [
- "type",
- "href"
- ],
- "properties": {
- "type": {
- "const": "url"
- },
- "text": {
- "$ref": "#/definitions/text_value"
- },
- "href": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "http://mysite.com"
- ]
- }
- }
- },
- "code": {
- "type": "object",
- "description": "A codeblock",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "code"
- },
- "value": {
- "type": "string"
- },
- "lang": {
- "type": "string",
- "description": "A programming language"
- }
- }
- },
- "value": {
- "type": "object",
- "description": "A field that can store a range of types of value",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "value"
- },
- "value": {
- "type": [
- "number",
- "string",
- "boolean"
- ]
- }
- }
- },
- "diff": {
- "type": "object",
- "description": "A diff",
- "required": [
- "type",
- "before",
- "after"
- ],
- "properties": {
- "type": {
- "const": "diff"
- },
- "before": {
- "type": "string"
- },
- "after": {
- "type": "string"
- }
- }
- },
- "markdown": {
- "type": "object",
- "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "markdown"
- },
- "value": {
- "$ref": "#/definitions/text_value",
- "examples": [
- "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
- ]
- }
- }
- },
- "commit": {
- "type": "object",
- "description": "A commit/tag/branch within the GitLab project",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "commit"
- },
- "value": {
- "type": "string",
- "description": "The commit SHA",
- "minLength": 1
- }
- }
- },
- "file_location": {
- "type": "object",
- "description": "A location within a file in the project",
- "required": [
- "type",
- "file_name",
- "line_start"
- ],
- "properties": {
- "type": {
- "const": "file-location"
- },
- "file_name": {
- "type": "string",
- "minLength": 1
- },
- "line_start": {
- "type": "integer"
- },
- "line_end": {
- "type": "integer"
- }
- }
- },
- "module_location": {
- "type": "object",
- "description": "A location within a binary module of the form module+relative_offset",
- "required": [
- "type",
- "module_name",
- "offset"
- ],
- "properties": {
- "type": {
- "const": "module-location"
- },
- "module_name": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "compiled_binary"
- ]
- },
- "offset": {
- "type": "integer",
- "examples": [
- 100
- ]
- }
- }
- }
- },
- "self": {
- "version": "14.1.0"
- },
- "required": [
- "version",
- "vulnerabilities"
- ],
- "additionalProperties": true,
- "properties": {
- "scan": {
- "type": "object",
- "required": [
- "end_time",
- "scanner",
- "start_time",
- "status",
- "type"
- ],
- "properties": {
- "end_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-01-28T03:26:02"
- ]
- },
- "messages": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Communication intended for the initiator of a scan.",
- "required": [
- "level",
- "value"
- ],
- "properties": {
- "level": {
- "type": "string",
- "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
- "enum": [
- "info",
- "warn",
- "fatal"
- ],
- "examples": [
- "info"
- ]
- },
- "value": {
- "type": "string",
- "description": "The message to communicate.",
- "minLength": 1,
- "examples": [
- "Permission denied, scanning aborted"
- ]
- }
- }
- }
- },
- "analyzer": {
- "type": "object",
- "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the analyzer.",
- "minLength": 1,
- "examples": [
- "gitlab-dast"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the analyzer, not required to be unique.",
- "minLength": 1,
- "examples": [
- "GitLab DAST"
- ]
- },
- "url": {
- "type": "string",
- "format": "uri",
- "pattern": "^https?://.+",
- "description": "A link to more information about the analyzer.",
- "examples": [
- "https://docs.gitlab.com/ee/user/application_security/dast"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the analyzer.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- },
- "version": {
- "type": "string",
- "description": "The version of the analyzer.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- }
- }
- },
- "scanner": {
- "type": "object",
- "description": "Object defining the scanner used to perform the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the scanner.",
- "minLength": 1,
- "examples": [
- "my-sast-scanner"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the scanner, not required to be unique.",
- "minLength": 1,
- "examples": [
- "My SAST Scanner"
- ]
- },
- "url": {
- "type": "string",
- "description": "A link to more information about the scanner.",
- "examples": [
- "https://scanner.url"
- ]
- },
- "version": {
- "type": "string",
- "description": "The version of the scanner.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the scanner.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- }
- }
- },
- "start_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-02-14T16:01:59"
- ]
- },
- "status": {
- "type": "string",
- "description": "Result of the scan.",
- "enum": [
- "success",
- "failure"
- ]
- },
- "type": {
- "type": "string",
- "description": "Type of the scan.",
- "enum": [
- "container_scanning"
- ]
- }
- }
- },
- "schema": {
- "type": "string",
- "description": "URI pointing to the validating security report schema.",
- "format": "uri"
- },
- "version": {
- "type": "string",
- "description": "The version of the schema to which the JSON report conforms.",
- "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
- },
- "vulnerabilities": {
- "type": "array",
- "description": "Array of vulnerability objects.",
- "items": {
- "type": "object",
- "description": "Describes the vulnerability using GitLab Flavored Markdown",
- "required": [
- "category",
- "cve",
- "identifiers",
- "location",
- "scanner"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
- "examples": [
- "642735a5-1425-428d-8d4e-3c854885a3c9"
- ]
- },
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
- "name": {
- "type": "string",
- "description": "The name of the vulnerability. This must not include the finding's specific information."
- },
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
- "description": {
- "type": "string",
- "description": "A long text section describing the vulnerability more fully."
- },
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
- "severity": {
- "type": "string",
- "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Info",
- "Unknown",
- "Low",
- "Medium",
- "High",
- "Critical"
- ]
- },
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
- "solution": {
- "type": "string",
- "description": "Explanation of how to fix the vulnerability."
- },
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
- "identifiers": {
- "type": "array",
- "minItems": 1,
- "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
- "items": {
- "type": "object",
- "required": [
- "type",
- "name",
- "value"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
- "minLength": 1
- },
- "name": {
- "type": "string",
- "description": "Human-readable name of the identifier.",
- "minLength": 1
- },
- "url": {
- "type": "string",
- "description": "URL of the identifier's documentation.",
- "format": "uri"
- },
- "value": {
- "type": "string",
- "description": "Value of the identifier, for matching purpose.",
- "minLength": 1
- }
- }
- }
- },
- "links": {
- "type": "array",
- "description": "An array of references to external documentation or articles that describe the vulnerability.",
- "items": {
- "type": "object",
- "required": [
- "url"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the vulnerability details link."
- },
- "url": {
- "type": "string",
- "description": "URL of the vulnerability details document.",
- "format": "uri"
- }
- }
- }
- },
- "details": {
- "$ref": "#/definitions/named_list/properties/items"
- },
- "tracking": {
- "description": "Describes how this vulnerability should be tracked as the project changes.",
- "oneOf": [
- {
- "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
- "required": [
- "items"
- ],
- "properties": {
- "type": {
- "const": "source"
- },
- "items": {
- "type": "array",
- "items": {
- "description": "An item that should be tracked using source-specific tracking methods.",
- "type": "object",
- "required": [
- "signatures"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the file that includes the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the file that includes the vulnerability."
- },
- "signatures": {
- "type": "array",
- "description": "An array of calculated tracking signatures for this tracking item.",
- "minItems": 1,
- "items": {
- "description": "A calculated tracking signature value and metadata.",
- "required": [
- "algorithm",
- "value"
- ],
- "properties": {
- "algorithm": {
- "type": "string",
- "description": "The algorithm used to generate the signature."
- },
- "value": {
- "type": "string",
- "description": "The result of this signature algorithm."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "Each tracking type must declare its own type."
- }
- }
- },
- "flags": {
- "description": "Flags that can be attached to vulnerabilities.",
- "type": "array",
- "items": {
- "type": "object",
- "description": "Informational flags identified and assigned to a vulnerability.",
- "required": [
- "type",
- "origin",
- "description"
- ],
- "properties": {
- "type": {
- "type": "string",
- "minLength": 1,
- "description": "Result of the scan.",
- "enum": [
- "flagged-as-likely-false-positive"
- ]
- },
- "origin": {
- "minLength": 1,
- "description": "Tool that issued the flag.",
- "type": "string"
- },
- "description": {
- "minLength": 1,
- "description": "What the flag is about.",
- "type": "string"
- }
- }
- }
- },
- "location": {
- "type": "object",
- "description": "Identifies the vulnerability's location.",
- "required": [
- "dependency",
- "operating_system",
- "image"
- ],
- "properties": {
- "dependency": {
- "type": "object",
- "description": "Describes the dependency of a project where the vulnerability is located.",
- "properties": {
- "package": {
- "type": "object",
- "description": "Provides information on the package where the vulnerability is located.",
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the package where the vulnerability is located."
- }
- }
- },
- "version": {
- "type": "string",
- "description": "Version of the vulnerable package."
- },
- "iid": {
- "description": "ID that identifies the dependency in the scope of a dependency file.",
- "type": "number"
- },
- "direct": {
- "type": "boolean",
- "description": "Tells whether this is a direct, top-level dependency of the scanned project."
- },
- "dependency_path": {
- "type": "array",
- "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.",
- "items": {
- "type": "object",
- "required": [
- "iid"
- ],
- "properties": {
- "iid": {
- "type": "number",
- "description": "ID that is unique in the scope of a parent object, and specific to the resource type."
- }
- }
- }
- }
- }
- },
- "operating_system": {
- "type": "string",
- "minLength": 1,
- "description": "The operating system that contains the vulnerable package."
- },
- "image": {
- "type": "string",
- "minLength": 1,
- "pattern": "^[^:]+(:\\d+[^:]*)?:[^:]+$",
- "description": "The analyzed Docker image."
- },
- "default_branch_image": {
- "type": "string",
- "maxLength": 255,
- "pattern": "^[a-zA-Z0-9/_.-]+(:\\d+[a-zA-Z0-9/_.-]*)?:[a-zA-Z0-9_.-]+$",
- "description": "The name of the image on the default branch."
- }
- }
- }
- }
- }
- },
- "remediations": {
- "type": "array",
- "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
- "items": {
- "type": "object",
- "required": [
- "fixes",
- "summary",
- "diff"
- ],
- "properties": {
- "fixes": {
- "type": "array",
- "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
- "items": {
- "type": "object",
- "required": [
- "cve"
- ],
- "properties": {
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- }
- }
- }
- },
- "summary": {
- "type": "string",
- "minLength": 1,
- "description": "An overview of how the vulnerabilities were fixed."
- },
- "diff": {
- "type": "string",
- "minLength": 1,
- "description": "A base64-encoded remediation code diff, compatible with git apply."
- }
- }
- }
- }
- }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.0/dast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.0/dast-report-format.json
deleted file mode 100644
index 73c03082d32..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.0/dast-report-format.json
+++ /dev/null
@@ -1,1291 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Report format for GitLab DAST",
- "description": "This schema provides the the report format for Dynamic Application Security Testing (https://docs.gitlab.com/ee/user/application_security/dast).",
- "definitions": {
- "detail_type": {
- "oneOf": [
- {
- "$ref": "#/definitions/named_list"
- },
- {
- "$ref": "#/definitions/list"
- },
- {
- "$ref": "#/definitions/table"
- },
- {
- "$ref": "#/definitions/text"
- },
- {
- "$ref": "#/definitions/url"
- },
- {
- "$ref": "#/definitions/code"
- },
- {
- "$ref": "#/definitions/value"
- },
- {
- "$ref": "#/definitions/diff"
- },
- {
- "$ref": "#/definitions/markdown"
- },
- {
- "$ref": "#/definitions/commit"
- },
- {
- "$ref": "#/definitions/file_location"
- },
- {
- "$ref": "#/definitions/module_location"
- }
- ]
- },
- "text_value": {
- "type": "string"
- },
- "named_field": {
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "$ref": "#/definitions/text_value",
- "minLength": 1
- },
- "description": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "named_list": {
- "type": "object",
- "description": "An object with named and typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "named-list"
- },
- "items": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- {
- "$ref": "#/definitions/named_field"
- },
- {
- "$ref": "#/definitions/detail_type"
- }
- ]
- }
- }
- }
- }
- },
- "list": {
- "type": "object",
- "description": "A list of typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- },
- "table": {
- "type": "object",
- "description": "A table of typed fields",
- "required": [
- "type",
- "rows"
- ],
- "properties": {
- "type": {
- "const": "table"
- },
- "header": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- },
- "rows": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- }
- },
- "text": {
- "type": "object",
- "description": "Raw text",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "text"
- },
- "value": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "url": {
- "type": "object",
- "description": "A single URL",
- "required": [
- "type",
- "href"
- ],
- "properties": {
- "type": {
- "const": "url"
- },
- "text": {
- "$ref": "#/definitions/text_value"
- },
- "href": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "http://mysite.com"
- ]
- }
- }
- },
- "code": {
- "type": "object",
- "description": "A codeblock",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "code"
- },
- "value": {
- "type": "string"
- },
- "lang": {
- "type": "string",
- "description": "A programming language"
- }
- }
- },
- "value": {
- "type": "object",
- "description": "A field that can store a range of types of value",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "value"
- },
- "value": {
- "type": [
- "number",
- "string",
- "boolean"
- ]
- }
- }
- },
- "diff": {
- "type": "object",
- "description": "A diff",
- "required": [
- "type",
- "before",
- "after"
- ],
- "properties": {
- "type": {
- "const": "diff"
- },
- "before": {
- "type": "string"
- },
- "after": {
- "type": "string"
- }
- }
- },
- "markdown": {
- "type": "object",
- "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "markdown"
- },
- "value": {
- "$ref": "#/definitions/text_value",
- "examples": [
- "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
- ]
- }
- }
- },
- "commit": {
- "type": "object",
- "description": "A commit/tag/branch within the GitLab project",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "commit"
- },
- "value": {
- "type": "string",
- "description": "The commit SHA",
- "minLength": 1
- }
- }
- },
- "file_location": {
- "type": "object",
- "description": "A location within a file in the project",
- "required": [
- "type",
- "file_name",
- "line_start"
- ],
- "properties": {
- "type": {
- "const": "file-location"
- },
- "file_name": {
- "type": "string",
- "minLength": 1
- },
- "line_start": {
- "type": "integer"
- },
- "line_end": {
- "type": "integer"
- }
- }
- },
- "module_location": {
- "type": "object",
- "description": "A location within a binary module of the form module+relative_offset",
- "required": [
- "type",
- "module_name",
- "offset"
- ],
- "properties": {
- "type": {
- "const": "module-location"
- },
- "module_name": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "compiled_binary"
- ]
- },
- "offset": {
- "type": "integer",
- "examples": [
- 100
- ]
- }
- }
- }
- },
- "self": {
- "version": "14.1.0"
- },
- "required": [
- "version",
- "vulnerabilities"
- ],
- "additionalProperties": true,
- "properties": {
- "scan": {
- "type": "object",
- "required": [
- "end_time",
- "scanned_resources",
- "scanner",
- "start_time",
- "status",
- "type"
- ],
- "properties": {
- "end_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-01-28T03:26:02"
- ]
- },
- "messages": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Communication intended for the initiator of a scan.",
- "required": [
- "level",
- "value"
- ],
- "properties": {
- "level": {
- "type": "string",
- "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
- "enum": [
- "info",
- "warn",
- "fatal"
- ],
- "examples": [
- "info"
- ]
- },
- "value": {
- "type": "string",
- "description": "The message to communicate.",
- "minLength": 1,
- "examples": [
- "Permission denied, scanning aborted"
- ]
- }
- }
- }
- },
- "analyzer": {
- "type": "object",
- "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the analyzer.",
- "minLength": 1,
- "examples": [
- "gitlab-dast"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the analyzer, not required to be unique.",
- "minLength": 1,
- "examples": [
- "GitLab DAST"
- ]
- },
- "url": {
- "type": "string",
- "format": "uri",
- "pattern": "^https?://.+",
- "description": "A link to more information about the analyzer.",
- "examples": [
- "https://docs.gitlab.com/ee/user/application_security/dast"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the analyzer.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- },
- "version": {
- "type": "string",
- "description": "The version of the analyzer.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- }
- }
- },
- "scanner": {
- "type": "object",
- "description": "Object defining the scanner used to perform the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the scanner.",
- "minLength": 1,
- "examples": [
- "my-sast-scanner"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the scanner, not required to be unique.",
- "minLength": 1,
- "examples": [
- "My SAST Scanner"
- ]
- },
- "url": {
- "type": "string",
- "description": "A link to more information about the scanner.",
- "examples": [
- "https://scanner.url"
- ]
- },
- "version": {
- "type": "string",
- "description": "The version of the scanner.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the scanner.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- }
- }
- },
- "start_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-02-14T16:01:59"
- ]
- },
- "status": {
- "type": "string",
- "description": "Result of the scan.",
- "enum": [
- "success",
- "failure"
- ]
- },
- "type": {
- "type": "string",
- "description": "Type of the scan.",
- "enum": [
- "dast",
- "api_fuzzing"
- ]
- },
- "scanned_resources": {
- "type": "array",
- "description": "The attack surface scanned by DAST.",
- "items": {
- "type": "object",
- "required": [
- "method",
- "url",
- "type"
- ],
- "properties": {
- "method": {
- "type": "string",
- "minLength": 1,
- "description": "HTTP method of the scanned resource.",
- "examples": [
- "GET",
- "POST",
- "HEAD"
- ]
- },
- "url": {
- "type": "string",
- "minLength": 1,
- "description": "URL of the scanned resource.",
- "examples": [
- "http://my.site.com/a-page"
- ]
- },
- "type": {
- "type": "string",
- "minLength": 1,
- "description": "Type of the scanned resource, for DAST, this must be 'url'.",
- "examples": [
- "url"
- ]
- }
- }
- }
- }
- }
- },
- "schema": {
- "type": "string",
- "description": "URI pointing to the validating security report schema.",
- "format": "uri"
- },
- "version": {
- "type": "string",
- "description": "The version of the schema to which the JSON report conforms.",
- "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
- },
- "vulnerabilities": {
- "type": "array",
- "description": "Array of vulnerability objects.",
- "items": {
- "type": "object",
- "description": "Describes the vulnerability using GitLab Flavored Markdown",
- "required": [
- "category",
- "cve",
- "identifiers",
- "location",
- "scanner"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
- "examples": [
- "642735a5-1425-428d-8d4e-3c854885a3c9"
- ]
- },
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
- "name": {
- "type": "string",
- "description": "The name of the vulnerability. This must not include the finding's specific information."
- },
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
- "description": {
- "type": "string",
- "description": "A long text section describing the vulnerability more fully."
- },
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
- "severity": {
- "type": "string",
- "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Info",
- "Unknown",
- "Low",
- "Medium",
- "High",
- "Critical"
- ]
- },
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
- "solution": {
- "type": "string",
- "description": "Explanation of how to fix the vulnerability."
- },
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
- "identifiers": {
- "type": "array",
- "minItems": 1,
- "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
- "items": {
- "type": "object",
- "required": [
- "type",
- "name",
- "value"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
- "minLength": 1
- },
- "name": {
- "type": "string",
- "description": "Human-readable name of the identifier.",
- "minLength": 1
- },
- "url": {
- "type": "string",
- "description": "URL of the identifier's documentation.",
- "format": "uri"
- },
- "value": {
- "type": "string",
- "description": "Value of the identifier, for matching purpose.",
- "minLength": 1
- }
- }
- }
- },
- "links": {
- "type": "array",
- "description": "An array of references to external documentation or articles that describe the vulnerability.",
- "items": {
- "type": "object",
- "required": [
- "url"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the vulnerability details link."
- },
- "url": {
- "type": "string",
- "description": "URL of the vulnerability details document.",
- "format": "uri"
- }
- }
- }
- },
- "details": {
- "$ref": "#/definitions/named_list/properties/items"
- },
- "tracking": {
- "description": "Describes how this vulnerability should be tracked as the project changes.",
- "oneOf": [
- {
- "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
- "required": [
- "items"
- ],
- "properties": {
- "type": {
- "const": "source"
- },
- "items": {
- "type": "array",
- "items": {
- "description": "An item that should be tracked using source-specific tracking methods.",
- "type": "object",
- "required": [
- "signatures"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the file that includes the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the file that includes the vulnerability."
- },
- "signatures": {
- "type": "array",
- "description": "An array of calculated tracking signatures for this tracking item.",
- "minItems": 1,
- "items": {
- "description": "A calculated tracking signature value and metadata.",
- "required": [
- "algorithm",
- "value"
- ],
- "properties": {
- "algorithm": {
- "type": "string",
- "description": "The algorithm used to generate the signature."
- },
- "value": {
- "type": "string",
- "description": "The result of this signature algorithm."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "Each tracking type must declare its own type."
- }
- }
- },
- "flags": {
- "description": "Flags that can be attached to vulnerabilities.",
- "type": "array",
- "items": {
- "type": "object",
- "description": "Informational flags identified and assigned to a vulnerability.",
- "required": [
- "type",
- "origin",
- "description"
- ],
- "properties": {
- "type": {
- "type": "string",
- "minLength": 1,
- "description": "Result of the scan.",
- "enum": [
- "flagged-as-likely-false-positive"
- ]
- },
- "origin": {
- "minLength": 1,
- "description": "Tool that issued the flag.",
- "type": "string"
- },
- "description": {
- "minLength": 1,
- "description": "What the flag is about.",
- "type": "string"
- }
- }
- }
- },
- "evidence": {
- "type": "object",
- "properties": {
- "source": {
- "type": "object",
- "description": "Source of evidence",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "Unique source identifier",
- "examples": [
- "assert:LogAnalysis",
- "assert:StatusCode"
- ]
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Source display name",
- "examples": [
- "Log Analysis",
- "Status Code"
- ]
- },
- "url": {
- "type": "string",
- "description": "Link to additional information",
- "examples": [
- "https://docs.gitlab.com/ee/development/integrations/secure.html"
- ]
- }
- }
- },
- "summary": {
- "type": "string",
- "description": "Human readable string containing evidence of the vulnerability.",
- "examples": [
- "Credit card 4111111111111111 found",
- "Server leaked information nginx/1.17.6"
- ]
- },
- "request": {
- "type": "object",
- "description": "An HTTP request.",
- "required": [
- "headers",
- "method",
- "url"
- ],
- "properties": {
- "headers": {
- "type": "array",
- "description": "HTTP headers present on the request.",
- "items": {
- "type": "object",
- "required": [
- "name",
- "value"
- ],
- "properties": {
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Name of the HTTP header.",
- "examples": [
- "Accept",
- "Content-Length",
- "Content-Type"
- ]
- },
- "value": {
- "type": "string",
- "minLength": 1,
- "description": "Value of the HTTP header.",
- "examples": [
- "*/*",
- "560",
- "application/json; charset=utf-8"
- ]
- }
- }
- }
- },
- "method": {
- "type": "string",
- "minLength": 1,
- "description": "HTTP method used in the request.",
- "examples": [
- "GET",
- "POST"
- ]
- },
- "url": {
- "type": "string",
- "minLength": 1,
- "description": "URL of the request.",
- "examples": [
- "http://my.site.com/vulnerable-endpoint?show-credit-card"
- ]
- },
- "body": {
- "type": "string",
- "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.",
- "examples": [
- "user=jsmith&first=%27&last=smith"
- ]
- }
- }
- },
- "response": {
- "type": "object",
- "description": "An HTTP response.",
- "required": [
- "headers",
- "reason_phrase",
- "status_code"
- ],
- "properties": {
- "headers": {
- "type": "array",
- "description": "HTTP headers present on the request.",
- "items": {
- "type": "object",
- "required": [
- "name",
- "value"
- ],
- "properties": {
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Name of the HTTP header.",
- "examples": [
- "Accept",
- "Content-Length",
- "Content-Type"
- ]
- },
- "value": {
- "type": "string",
- "minLength": 1,
- "description": "Value of the HTTP header.",
- "examples": [
- "*/*",
- "560",
- "application/json; charset=utf-8"
- ]
- }
- }
- }
- },
- "reason_phrase": {
- "type": "string",
- "description": "HTTP reason phrase of the response.",
- "examples": [
- "OK",
- "Internal Server Error"
- ]
- },
- "status_code": {
- "type": "integer",
- "description": "HTTP status code of the response.",
- "examples": [
- 200,
- 500
- ]
- },
- "body": {
- "type": "string",
- "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.",
- "examples": [
- "{\"user_id\": 2}"
- ]
- }
- }
- },
- "supporting_messages": {
- "type": "array",
- "description": "Array of supporting http messages.",
- "items": {
- "type": "object",
- "description": "A supporting http message.",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Message display name.",
- "examples": [
- "Unmodified",
- "Recorded"
- ]
- },
- "request": {
- "type": "object",
- "description": "An HTTP request.",
- "required": [
- "headers",
- "method",
- "url"
- ],
- "properties": {
- "headers": {
- "type": "array",
- "description": "HTTP headers present on the request.",
- "items": {
- "type": "object",
- "required": [
- "name",
- "value"
- ],
- "properties": {
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Name of the HTTP header.",
- "examples": [
- "Accept",
- "Content-Length",
- "Content-Type"
- ]
- },
- "value": {
- "type": "string",
- "minLength": 1,
- "description": "Value of the HTTP header.",
- "examples": [
- "*/*",
- "560",
- "application/json; charset=utf-8"
- ]
- }
- }
- }
- },
- "method": {
- "type": "string",
- "minLength": 1,
- "description": "HTTP method used in the request.",
- "examples": [
- "GET",
- "POST"
- ]
- },
- "url": {
- "type": "string",
- "minLength": 1,
- "description": "URL of the request.",
- "examples": [
- "http://my.site.com/vulnerable-endpoint?show-credit-card"
- ]
- },
- "body": {
- "type": "string",
- "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.",
- "examples": [
- "user=jsmith&first=%27&last=smith"
- ]
- }
- }
- },
- "response": {
- "type": "object",
- "description": "An HTTP response.",
- "required": [
- "headers",
- "reason_phrase",
- "status_code"
- ],
- "properties": {
- "headers": {
- "type": "array",
- "description": "HTTP headers present on the request.",
- "items": {
- "type": "object",
- "required": [
- "name",
- "value"
- ],
- "properties": {
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Name of the HTTP header.",
- "examples": [
- "Accept",
- "Content-Length",
- "Content-Type"
- ]
- },
- "value": {
- "type": "string",
- "minLength": 1,
- "description": "Value of the HTTP header.",
- "examples": [
- "*/*",
- "560",
- "application/json; charset=utf-8"
- ]
- }
- }
- }
- },
- "reason_phrase": {
- "type": "string",
- "description": "HTTP reason phrase of the response.",
- "examples": [
- "OK",
- "Internal Server Error"
- ]
- },
- "status_code": {
- "type": "integer",
- "description": "HTTP status code of the response.",
- "examples": [
- 200,
- 500
- ]
- },
- "body": {
- "type": "string",
- "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.",
- "examples": [
- "{\"user_id\": 2}"
- ]
- }
- }
- }
- }
- }
- }
- }
- },
- "location": {
- "type": "object",
- "description": "Identifies the vulnerability's location.",
- "properties": {
- "hostname": {
- "type": "string",
- "description": "The protocol, domain, and port of the application where the vulnerability was found."
- },
- "method": {
- "type": "string",
- "description": "The HTTP method that was used to request the URL where the vulnerability was found."
- },
- "param": {
- "type": "string",
- "description": "A value provided by a vulnerability rule related to the found vulnerability. Examples include a header value, or a parameter used in a HTTP POST."
- },
- "path": {
- "type": "string",
- "description": "The path of the URL where the vulnerability was found. Typically, this would start with a forward slash."
- }
- }
- },
- "assets": {
- "type": "array",
- "description": "Array of build assets associated with vulnerability.",
- "items": {
- "type": "object",
- "description": "Describes an asset associated with vulnerability.",
- "required": [
- "type",
- "name",
- "url"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "The type of asset",
- "enum": [
- "http_session",
- "postman"
- ]
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Display name for asset",
- "examples": [
- "HTTP Messages",
- "Postman Collection"
- ]
- },
- "url": {
- "type": "string",
- "minLength": 1,
- "description": "Link to asset in build artifacts",
- "examples": [
- "https://gitlab.com/gitlab-org/security-products/dast/-/jobs/626397001/artifacts/file//output/zap_session.data"
- ]
- }
- }
- }
- },
- "discovered_at": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss.sss, representing when the vulnerability was discovered",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}\\.\\d{3}$",
- "examples": [
- "2020-01-28T03:26:02.956"
- ]
- }
- }
- }
- },
- "remediations": {
- "type": "array",
- "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
- "items": {
- "type": "object",
- "required": [
- "fixes",
- "summary",
- "diff"
- ],
- "properties": {
- "fixes": {
- "type": "array",
- "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
- "items": {
- "type": "object",
- "required": [
- "cve"
- ],
- "properties": {
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- }
- }
- }
- },
- "summary": {
- "type": "string",
- "minLength": 1,
- "description": "An overview of how the vulnerabilities were fixed."
- },
- "diff": {
- "type": "string",
- "minLength": 1,
- "description": "A base64-encoded remediation code diff, compatible with git apply."
- }
- }
- }
- }
- }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/container-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/container-scanning-report-format.json
deleted file mode 100644
index a13e0418499..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/container-scanning-report-format.json
+++ /dev/null
@@ -1,911 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Report format for GitLab Container Scanning",
- "description": "This schema provides the the report format for Container Scanning (https://docs.gitlab.com/ee/user/application_security/container_scanning).",
- "definitions": {
- "detail_type": {
- "oneOf": [
- {
- "$ref": "#/definitions/named_list"
- },
- {
- "$ref": "#/definitions/list"
- },
- {
- "$ref": "#/definitions/table"
- },
- {
- "$ref": "#/definitions/text"
- },
- {
- "$ref": "#/definitions/url"
- },
- {
- "$ref": "#/definitions/code"
- },
- {
- "$ref": "#/definitions/value"
- },
- {
- "$ref": "#/definitions/diff"
- },
- {
- "$ref": "#/definitions/markdown"
- },
- {
- "$ref": "#/definitions/commit"
- },
- {
- "$ref": "#/definitions/file_location"
- },
- {
- "$ref": "#/definitions/module_location"
- }
- ]
- },
- "text_value": {
- "type": "string"
- },
- "named_field": {
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "$ref": "#/definitions/text_value",
- "minLength": 1
- },
- "description": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "named_list": {
- "type": "object",
- "description": "An object with named and typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "named-list"
- },
- "items": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- {
- "$ref": "#/definitions/named_field"
- },
- {
- "$ref": "#/definitions/detail_type"
- }
- ]
- }
- }
- }
- }
- },
- "list": {
- "type": "object",
- "description": "A list of typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- },
- "table": {
- "type": "object",
- "description": "A table of typed fields",
- "required": [
- "type",
- "rows"
- ],
- "properties": {
- "type": {
- "const": "table"
- },
- "header": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- },
- "rows": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- }
- },
- "text": {
- "type": "object",
- "description": "Raw text",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "text"
- },
- "value": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "url": {
- "type": "object",
- "description": "A single URL",
- "required": [
- "type",
- "href"
- ],
- "properties": {
- "type": {
- "const": "url"
- },
- "text": {
- "$ref": "#/definitions/text_value"
- },
- "href": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "http://mysite.com"
- ]
- }
- }
- },
- "code": {
- "type": "object",
- "description": "A codeblock",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "code"
- },
- "value": {
- "type": "string"
- },
- "lang": {
- "type": "string",
- "description": "A programming language"
- }
- }
- },
- "value": {
- "type": "object",
- "description": "A field that can store a range of types of value",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "value"
- },
- "value": {
- "type": [
- "number",
- "string",
- "boolean"
- ]
- }
- }
- },
- "diff": {
- "type": "object",
- "description": "A diff",
- "required": [
- "type",
- "before",
- "after"
- ],
- "properties": {
- "type": {
- "const": "diff"
- },
- "before": {
- "type": "string"
- },
- "after": {
- "type": "string"
- }
- }
- },
- "markdown": {
- "type": "object",
- "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "markdown"
- },
- "value": {
- "$ref": "#/definitions/text_value",
- "examples": [
- "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
- ]
- }
- }
- },
- "commit": {
- "type": "object",
- "description": "A commit/tag/branch within the GitLab project",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "commit"
- },
- "value": {
- "type": "string",
- "description": "The commit SHA",
- "minLength": 1
- }
- }
- },
- "file_location": {
- "type": "object",
- "description": "A location within a file in the project",
- "required": [
- "type",
- "file_name",
- "line_start"
- ],
- "properties": {
- "type": {
- "const": "file-location"
- },
- "file_name": {
- "type": "string",
- "minLength": 1
- },
- "line_start": {
- "type": "integer"
- },
- "line_end": {
- "type": "integer"
- }
- }
- },
- "module_location": {
- "type": "object",
- "description": "A location within a binary module of the form module+relative_offset",
- "required": [
- "type",
- "module_name",
- "offset"
- ],
- "properties": {
- "type": {
- "const": "module-location"
- },
- "module_name": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "compiled_binary"
- ]
- },
- "offset": {
- "type": "integer",
- "examples": [
- 100
- ]
- }
- }
- }
- },
- "self": {
- "version": "14.1.1"
- },
- "required": [
- "version",
- "vulnerabilities"
- ],
- "additionalProperties": true,
- "properties": {
- "scan": {
- "type": "object",
- "required": [
- "end_time",
- "scanner",
- "start_time",
- "status",
- "type"
- ],
- "properties": {
- "end_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-01-28T03:26:02"
- ]
- },
- "messages": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Communication intended for the initiator of a scan.",
- "required": [
- "level",
- "value"
- ],
- "properties": {
- "level": {
- "type": "string",
- "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
- "enum": [
- "info",
- "warn",
- "fatal"
- ],
- "examples": [
- "info"
- ]
- },
- "value": {
- "type": "string",
- "description": "The message to communicate.",
- "minLength": 1,
- "examples": [
- "Permission denied, scanning aborted"
- ]
- }
- }
- }
- },
- "analyzer": {
- "type": "object",
- "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the analyzer.",
- "minLength": 1,
- "examples": [
- "gitlab-dast"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the analyzer, not required to be unique.",
- "minLength": 1,
- "examples": [
- "GitLab DAST"
- ]
- },
- "url": {
- "type": "string",
- "format": "uri",
- "pattern": "^https?://.+",
- "description": "A link to more information about the analyzer.",
- "examples": [
- "https://docs.gitlab.com/ee/user/application_security/dast"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the analyzer.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- },
- "version": {
- "type": "string",
- "description": "The version of the analyzer.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- }
- }
- },
- "scanner": {
- "type": "object",
- "description": "Object defining the scanner used to perform the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the scanner.",
- "minLength": 1,
- "examples": [
- "my-sast-scanner"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the scanner, not required to be unique.",
- "minLength": 1,
- "examples": [
- "My SAST Scanner"
- ]
- },
- "url": {
- "type": "string",
- "description": "A link to more information about the scanner.",
- "examples": [
- "https://scanner.url"
- ]
- },
- "version": {
- "type": "string",
- "description": "The version of the scanner.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the scanner.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- }
- }
- },
- "start_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-02-14T16:01:59"
- ]
- },
- "status": {
- "type": "string",
- "description": "Result of the scan.",
- "enum": [
- "success",
- "failure"
- ]
- },
- "type": {
- "type": "string",
- "description": "Type of the scan.",
- "enum": [
- "container_scanning"
- ]
- }
- }
- },
- "schema": {
- "type": "string",
- "description": "URI pointing to the validating security report schema.",
- "format": "uri"
- },
- "version": {
- "type": "string",
- "description": "The version of the schema to which the JSON report conforms.",
- "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
- },
- "vulnerabilities": {
- "type": "array",
- "description": "Array of vulnerability objects.",
- "items": {
- "type": "object",
- "description": "Describes the vulnerability using GitLab Flavored Markdown",
- "required": [
- "category",
- "cve",
- "identifiers",
- "location",
- "scanner"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
- "examples": [
- "642735a5-1425-428d-8d4e-3c854885a3c9"
- ]
- },
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
- "name": {
- "type": "string",
- "description": "The name of the vulnerability. This must not include the finding's specific information."
- },
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
- "description": {
- "type": "string",
- "description": "A long text section describing the vulnerability more fully."
- },
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
- "severity": {
- "type": "string",
- "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Info",
- "Unknown",
- "Low",
- "Medium",
- "High",
- "Critical"
- ]
- },
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
- "solution": {
- "type": "string",
- "description": "Explanation of how to fix the vulnerability."
- },
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
- "identifiers": {
- "type": "array",
- "minItems": 1,
- "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
- "items": {
- "type": "object",
- "required": [
- "type",
- "name",
- "value"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
- "minLength": 1
- },
- "name": {
- "type": "string",
- "description": "Human-readable name of the identifier.",
- "minLength": 1
- },
- "url": {
- "type": "string",
- "description": "URL of the identifier's documentation.",
- "format": "uri"
- },
- "value": {
- "type": "string",
- "description": "Value of the identifier, for matching purpose.",
- "minLength": 1
- }
- }
- }
- },
- "links": {
- "type": "array",
- "description": "An array of references to external documentation or articles that describe the vulnerability.",
- "items": {
- "type": "object",
- "required": [
- "url"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the vulnerability details link."
- },
- "url": {
- "type": "string",
- "description": "URL of the vulnerability details document.",
- "format": "uri"
- }
- }
- }
- },
- "details": {
- "$ref": "#/definitions/named_list/properties/items"
- },
- "tracking": {
- "description": "Describes how this vulnerability should be tracked as the project changes.",
- "oneOf": [
- {
- "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
- "required": [
- "items"
- ],
- "properties": {
- "type": {
- "const": "source"
- },
- "items": {
- "type": "array",
- "items": {
- "description": "An item that should be tracked using source-specific tracking methods.",
- "type": "object",
- "required": [
- "signatures"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the file that includes the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the file that includes the vulnerability."
- },
- "signatures": {
- "type": "array",
- "description": "An array of calculated tracking signatures for this tracking item.",
- "minItems": 1,
- "items": {
- "description": "A calculated tracking signature value and metadata.",
- "required": [
- "algorithm",
- "value"
- ],
- "properties": {
- "algorithm": {
- "type": "string",
- "description": "The algorithm used to generate the signature."
- },
- "value": {
- "type": "string",
- "description": "The result of this signature algorithm."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "Each tracking type must declare its own type."
- }
- }
- },
- "flags": {
- "description": "Flags that can be attached to vulnerabilities.",
- "type": "array",
- "items": {
- "type": "object",
- "description": "Informational flags identified and assigned to a vulnerability.",
- "required": [
- "type",
- "origin",
- "description"
- ],
- "properties": {
- "type": {
- "type": "string",
- "minLength": 1,
- "description": "Result of the scan.",
- "enum": [
- "flagged-as-likely-false-positive"
- ]
- },
- "origin": {
- "minLength": 1,
- "description": "Tool that issued the flag.",
- "type": "string"
- },
- "description": {
- "minLength": 1,
- "description": "What the flag is about.",
- "type": "string"
- }
- }
- }
- },
- "location": {
- "type": "object",
- "description": "Identifies the vulnerability's location.",
- "required": [
- "dependency",
- "operating_system",
- "image"
- ],
- "properties": {
- "dependency": {
- "type": "object",
- "description": "Describes the dependency of a project where the vulnerability is located.",
- "properties": {
- "package": {
- "type": "object",
- "description": "Provides information on the package where the vulnerability is located.",
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the package where the vulnerability is located."
- }
- }
- },
- "version": {
- "type": "string",
- "description": "Version of the vulnerable package."
- },
- "iid": {
- "description": "ID that identifies the dependency in the scope of a dependency file.",
- "type": "number"
- },
- "direct": {
- "type": "boolean",
- "description": "Tells whether this is a direct, top-level dependency of the scanned project."
- },
- "dependency_path": {
- "type": "array",
- "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.",
- "items": {
- "type": "object",
- "required": [
- "iid"
- ],
- "properties": {
- "iid": {
- "type": "number",
- "description": "ID that is unique in the scope of a parent object, and specific to the resource type."
- }
- }
- }
- }
- }
- },
- "operating_system": {
- "type": "string",
- "minLength": 1,
- "description": "The operating system that contains the vulnerable package."
- },
- "image": {
- "type": "string",
- "minLength": 1,
- "pattern": "^[^:]+(:\\d+[^:]*)?:[^:]+$",
- "description": "The analyzed Docker image."
- },
- "default_branch_image": {
- "type": "string",
- "maxLength": 255,
- "pattern": "^[a-zA-Z0-9/_.-]+(:\\d+[a-zA-Z0-9/_.-]*)?:[a-zA-Z0-9_.-]+$",
- "description": "The name of the image on the default branch."
- }
- }
- }
- }
- }
- },
- "remediations": {
- "type": "array",
- "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
- "items": {
- "type": "object",
- "required": [
- "fixes",
- "summary",
- "diff"
- ],
- "properties": {
- "fixes": {
- "type": "array",
- "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
- "items": {
- "type": "object",
- "required": [
- "cve"
- ],
- "properties": {
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- }
- }
- }
- },
- "summary": {
- "type": "string",
- "minLength": 1,
- "description": "An overview of how the vulnerabilities were fixed."
- },
- "diff": {
- "type": "string",
- "minLength": 1,
- "description": "A base64-encoded remediation code diff, compatible with git apply."
- }
- }
- }
- }
- }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/coverage-fuzzing-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/coverage-fuzzing-report-format.json
deleted file mode 100644
index 050c34669b3..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/coverage-fuzzing-report-format.json
+++ /dev/null
@@ -1,874 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Report format for GitLab Fuzz Testing",
- "description": "This schema provides the report format for Coverage Guided Fuzz Testing (https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing).",
- "definitions": {
- "detail_type": {
- "oneOf": [
- {
- "$ref": "#/definitions/named_list"
- },
- {
- "$ref": "#/definitions/list"
- },
- {
- "$ref": "#/definitions/table"
- },
- {
- "$ref": "#/definitions/text"
- },
- {
- "$ref": "#/definitions/url"
- },
- {
- "$ref": "#/definitions/code"
- },
- {
- "$ref": "#/definitions/value"
- },
- {
- "$ref": "#/definitions/diff"
- },
- {
- "$ref": "#/definitions/markdown"
- },
- {
- "$ref": "#/definitions/commit"
- },
- {
- "$ref": "#/definitions/file_location"
- },
- {
- "$ref": "#/definitions/module_location"
- }
- ]
- },
- "text_value": {
- "type": "string"
- },
- "named_field": {
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "$ref": "#/definitions/text_value",
- "minLength": 1
- },
- "description": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "named_list": {
- "type": "object",
- "description": "An object with named and typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "named-list"
- },
- "items": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- {
- "$ref": "#/definitions/named_field"
- },
- {
- "$ref": "#/definitions/detail_type"
- }
- ]
- }
- }
- }
- }
- },
- "list": {
- "type": "object",
- "description": "A list of typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- },
- "table": {
- "type": "object",
- "description": "A table of typed fields",
- "required": [
- "type",
- "rows"
- ],
- "properties": {
- "type": {
- "const": "table"
- },
- "header": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- },
- "rows": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- }
- },
- "text": {
- "type": "object",
- "description": "Raw text",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "text"
- },
- "value": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "url": {
- "type": "object",
- "description": "A single URL",
- "required": [
- "type",
- "href"
- ],
- "properties": {
- "type": {
- "const": "url"
- },
- "text": {
- "$ref": "#/definitions/text_value"
- },
- "href": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "http://mysite.com"
- ]
- }
- }
- },
- "code": {
- "type": "object",
- "description": "A codeblock",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "code"
- },
- "value": {
- "type": "string"
- },
- "lang": {
- "type": "string",
- "description": "A programming language"
- }
- }
- },
- "value": {
- "type": "object",
- "description": "A field that can store a range of types of value",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "value"
- },
- "value": {
- "type": [
- "number",
- "string",
- "boolean"
- ]
- }
- }
- },
- "diff": {
- "type": "object",
- "description": "A diff",
- "required": [
- "type",
- "before",
- "after"
- ],
- "properties": {
- "type": {
- "const": "diff"
- },
- "before": {
- "type": "string"
- },
- "after": {
- "type": "string"
- }
- }
- },
- "markdown": {
- "type": "object",
- "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "markdown"
- },
- "value": {
- "$ref": "#/definitions/text_value",
- "examples": [
- "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
- ]
- }
- }
- },
- "commit": {
- "type": "object",
- "description": "A commit/tag/branch within the GitLab project",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "commit"
- },
- "value": {
- "type": "string",
- "description": "The commit SHA",
- "minLength": 1
- }
- }
- },
- "file_location": {
- "type": "object",
- "description": "A location within a file in the project",
- "required": [
- "type",
- "file_name",
- "line_start"
- ],
- "properties": {
- "type": {
- "const": "file-location"
- },
- "file_name": {
- "type": "string",
- "minLength": 1
- },
- "line_start": {
- "type": "integer"
- },
- "line_end": {
- "type": "integer"
- }
- }
- },
- "module_location": {
- "type": "object",
- "description": "A location within a binary module of the form module+relative_offset",
- "required": [
- "type",
- "module_name",
- "offset"
- ],
- "properties": {
- "type": {
- "const": "module-location"
- },
- "module_name": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "compiled_binary"
- ]
- },
- "offset": {
- "type": "integer",
- "examples": [
- 100
- ]
- }
- }
- }
- },
- "self": {
- "version": "14.1.1"
- },
- "required": [
- "version",
- "vulnerabilities"
- ],
- "additionalProperties": true,
- "properties": {
- "scan": {
- "type": "object",
- "required": [
- "end_time",
- "scanner",
- "start_time",
- "status",
- "type"
- ],
- "properties": {
- "end_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-01-28T03:26:02"
- ]
- },
- "messages": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Communication intended for the initiator of a scan.",
- "required": [
- "level",
- "value"
- ],
- "properties": {
- "level": {
- "type": "string",
- "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
- "enum": [
- "info",
- "warn",
- "fatal"
- ],
- "examples": [
- "info"
- ]
- },
- "value": {
- "type": "string",
- "description": "The message to communicate.",
- "minLength": 1,
- "examples": [
- "Permission denied, scanning aborted"
- ]
- }
- }
- }
- },
- "analyzer": {
- "type": "object",
- "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the analyzer.",
- "minLength": 1,
- "examples": [
- "gitlab-dast"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the analyzer, not required to be unique.",
- "minLength": 1,
- "examples": [
- "GitLab DAST"
- ]
- },
- "url": {
- "type": "string",
- "format": "uri",
- "pattern": "^https?://.+",
- "description": "A link to more information about the analyzer.",
- "examples": [
- "https://docs.gitlab.com/ee/user/application_security/dast"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the analyzer.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- },
- "version": {
- "type": "string",
- "description": "The version of the analyzer.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- }
- }
- },
- "scanner": {
- "type": "object",
- "description": "Object defining the scanner used to perform the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the scanner.",
- "minLength": 1,
- "examples": [
- "my-sast-scanner"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the scanner, not required to be unique.",
- "minLength": 1,
- "examples": [
- "My SAST Scanner"
- ]
- },
- "url": {
- "type": "string",
- "description": "A link to more information about the scanner.",
- "examples": [
- "https://scanner.url"
- ]
- },
- "version": {
- "type": "string",
- "description": "The version of the scanner.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the scanner.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- }
- }
- },
- "start_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-02-14T16:01:59"
- ]
- },
- "status": {
- "type": "string",
- "description": "Result of the scan.",
- "enum": [
- "success",
- "failure"
- ]
- },
- "type": {
- "type": "string",
- "description": "Type of the scan.",
- "enum": [
- "coverage_fuzzing"
- ]
- }
- }
- },
- "schema": {
- "type": "string",
- "description": "URI pointing to the validating security report schema.",
- "format": "uri"
- },
- "version": {
- "type": "string",
- "description": "The version of the schema to which the JSON report conforms.",
- "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
- },
- "vulnerabilities": {
- "type": "array",
- "description": "Array of vulnerability objects.",
- "items": {
- "type": "object",
- "description": "Describes the vulnerability using GitLab Flavored Markdown",
- "required": [
- "category",
- "cve",
- "identifiers",
- "location",
- "scanner"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
- "examples": [
- "642735a5-1425-428d-8d4e-3c854885a3c9"
- ]
- },
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
- "name": {
- "type": "string",
- "description": "The name of the vulnerability. This must not include the finding's specific information."
- },
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
- "description": {
- "type": "string",
- "description": "A long text section describing the vulnerability more fully."
- },
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
- "severity": {
- "type": "string",
- "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Info",
- "Unknown",
- "Low",
- "Medium",
- "High",
- "Critical"
- ]
- },
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
- "solution": {
- "type": "string",
- "description": "Explanation of how to fix the vulnerability."
- },
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
- "identifiers": {
- "type": "array",
- "minItems": 1,
- "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
- "items": {
- "type": "object",
- "required": [
- "type",
- "name",
- "value"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
- "minLength": 1
- },
- "name": {
- "type": "string",
- "description": "Human-readable name of the identifier.",
- "minLength": 1
- },
- "url": {
- "type": "string",
- "description": "URL of the identifier's documentation.",
- "format": "uri"
- },
- "value": {
- "type": "string",
- "description": "Value of the identifier, for matching purpose.",
- "minLength": 1
- }
- }
- }
- },
- "links": {
- "type": "array",
- "description": "An array of references to external documentation or articles that describe the vulnerability.",
- "items": {
- "type": "object",
- "required": [
- "url"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the vulnerability details link."
- },
- "url": {
- "type": "string",
- "description": "URL of the vulnerability details document.",
- "format": "uri"
- }
- }
- }
- },
- "details": {
- "$ref": "#/definitions/named_list/properties/items"
- },
- "tracking": {
- "description": "Describes how this vulnerability should be tracked as the project changes.",
- "oneOf": [
- {
- "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
- "required": [
- "items"
- ],
- "properties": {
- "type": {
- "const": "source"
- },
- "items": {
- "type": "array",
- "items": {
- "description": "An item that should be tracked using source-specific tracking methods.",
- "type": "object",
- "required": [
- "signatures"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the file that includes the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the file that includes the vulnerability."
- },
- "signatures": {
- "type": "array",
- "description": "An array of calculated tracking signatures for this tracking item.",
- "minItems": 1,
- "items": {
- "description": "A calculated tracking signature value and metadata.",
- "required": [
- "algorithm",
- "value"
- ],
- "properties": {
- "algorithm": {
- "type": "string",
- "description": "The algorithm used to generate the signature."
- },
- "value": {
- "type": "string",
- "description": "The result of this signature algorithm."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "Each tracking type must declare its own type."
- }
- }
- },
- "flags": {
- "description": "Flags that can be attached to vulnerabilities.",
- "type": "array",
- "items": {
- "type": "object",
- "description": "Informational flags identified and assigned to a vulnerability.",
- "required": [
- "type",
- "origin",
- "description"
- ],
- "properties": {
- "type": {
- "type": "string",
- "minLength": 1,
- "description": "Result of the scan.",
- "enum": [
- "flagged-as-likely-false-positive"
- ]
- },
- "origin": {
- "minLength": 1,
- "description": "Tool that issued the flag.",
- "type": "string"
- },
- "description": {
- "minLength": 1,
- "description": "What the flag is about.",
- "type": "string"
- }
- }
- }
- },
- "location": {
- "description": "The location of the error",
- "type": "object",
- "properties": {
- "crash_address": {
- "type": "string",
- "description": "The relative address in memory were the crash occurred.",
- "examples": [
- "0xabababab"
- ]
- },
- "stacktrace_snippet": {
- "type": "string",
- "description": "The stack trace recorded during fuzzing resulting the crash.",
- "examples": [
- "func_a+0xabcd\nfunc_b+0xabcc"
- ]
- },
- "crash_state": {
- "type": "string",
- "description": "Minimised and normalized crash stack-trace (called crash_state).",
- "examples": [
- "func_a+0xa\nfunc_b+0xb\nfunc_c+0xc"
- ]
- },
- "crash_type": {
- "type": "string",
- "description": "Type of the crash.",
- "examples": [
- "Heap-Buffer-overflow",
- "Division-by-zero"
- ]
- }
- }
- }
- }
- }
- },
- "remediations": {
- "type": "array",
- "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
- "items": {
- "type": "object",
- "required": [
- "fixes",
- "summary",
- "diff"
- ],
- "properties": {
- "fixes": {
- "type": "array",
- "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
- "items": {
- "type": "object",
- "required": [
- "cve"
- ],
- "properties": {
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- }
- }
- }
- },
- "summary": {
- "type": "string",
- "minLength": 1,
- "description": "An overview of how the vulnerabilities were fixed."
- },
- "diff": {
- "type": "string",
- "minLength": 1,
- "description": "A base64-encoded remediation code diff, compatible with git apply."
- }
- }
- }
- }
- }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/dast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/dast-report-format.json
deleted file mode 100644
index 62ed293ad44..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/dast-report-format.json
+++ /dev/null
@@ -1,1291 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Report format for GitLab DAST",
- "description": "This schema provides the the report format for Dynamic Application Security Testing (https://docs.gitlab.com/ee/user/application_security/dast).",
- "definitions": {
- "detail_type": {
- "oneOf": [
- {
- "$ref": "#/definitions/named_list"
- },
- {
- "$ref": "#/definitions/list"
- },
- {
- "$ref": "#/definitions/table"
- },
- {
- "$ref": "#/definitions/text"
- },
- {
- "$ref": "#/definitions/url"
- },
- {
- "$ref": "#/definitions/code"
- },
- {
- "$ref": "#/definitions/value"
- },
- {
- "$ref": "#/definitions/diff"
- },
- {
- "$ref": "#/definitions/markdown"
- },
- {
- "$ref": "#/definitions/commit"
- },
- {
- "$ref": "#/definitions/file_location"
- },
- {
- "$ref": "#/definitions/module_location"
- }
- ]
- },
- "text_value": {
- "type": "string"
- },
- "named_field": {
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "$ref": "#/definitions/text_value",
- "minLength": 1
- },
- "description": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "named_list": {
- "type": "object",
- "description": "An object with named and typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "named-list"
- },
- "items": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- {
- "$ref": "#/definitions/named_field"
- },
- {
- "$ref": "#/definitions/detail_type"
- }
- ]
- }
- }
- }
- }
- },
- "list": {
- "type": "object",
- "description": "A list of typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- },
- "table": {
- "type": "object",
- "description": "A table of typed fields",
- "required": [
- "type",
- "rows"
- ],
- "properties": {
- "type": {
- "const": "table"
- },
- "header": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- },
- "rows": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- }
- },
- "text": {
- "type": "object",
- "description": "Raw text",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "text"
- },
- "value": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "url": {
- "type": "object",
- "description": "A single URL",
- "required": [
- "type",
- "href"
- ],
- "properties": {
- "type": {
- "const": "url"
- },
- "text": {
- "$ref": "#/definitions/text_value"
- },
- "href": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "http://mysite.com"
- ]
- }
- }
- },
- "code": {
- "type": "object",
- "description": "A codeblock",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "code"
- },
- "value": {
- "type": "string"
- },
- "lang": {
- "type": "string",
- "description": "A programming language"
- }
- }
- },
- "value": {
- "type": "object",
- "description": "A field that can store a range of types of value",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "value"
- },
- "value": {
- "type": [
- "number",
- "string",
- "boolean"
- ]
- }
- }
- },
- "diff": {
- "type": "object",
- "description": "A diff",
- "required": [
- "type",
- "before",
- "after"
- ],
- "properties": {
- "type": {
- "const": "diff"
- },
- "before": {
- "type": "string"
- },
- "after": {
- "type": "string"
- }
- }
- },
- "markdown": {
- "type": "object",
- "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "markdown"
- },
- "value": {
- "$ref": "#/definitions/text_value",
- "examples": [
- "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
- ]
- }
- }
- },
- "commit": {
- "type": "object",
- "description": "A commit/tag/branch within the GitLab project",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "commit"
- },
- "value": {
- "type": "string",
- "description": "The commit SHA",
- "minLength": 1
- }
- }
- },
- "file_location": {
- "type": "object",
- "description": "A location within a file in the project",
- "required": [
- "type",
- "file_name",
- "line_start"
- ],
- "properties": {
- "type": {
- "const": "file-location"
- },
- "file_name": {
- "type": "string",
- "minLength": 1
- },
- "line_start": {
- "type": "integer"
- },
- "line_end": {
- "type": "integer"
- }
- }
- },
- "module_location": {
- "type": "object",
- "description": "A location within a binary module of the form module+relative_offset",
- "required": [
- "type",
- "module_name",
- "offset"
- ],
- "properties": {
- "type": {
- "const": "module-location"
- },
- "module_name": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "compiled_binary"
- ]
- },
- "offset": {
- "type": "integer",
- "examples": [
- 100
- ]
- }
- }
- }
- },
- "self": {
- "version": "14.1.1"
- },
- "required": [
- "version",
- "vulnerabilities"
- ],
- "additionalProperties": true,
- "properties": {
- "scan": {
- "type": "object",
- "required": [
- "end_time",
- "scanned_resources",
- "scanner",
- "start_time",
- "status",
- "type"
- ],
- "properties": {
- "end_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-01-28T03:26:02"
- ]
- },
- "messages": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Communication intended for the initiator of a scan.",
- "required": [
- "level",
- "value"
- ],
- "properties": {
- "level": {
- "type": "string",
- "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
- "enum": [
- "info",
- "warn",
- "fatal"
- ],
- "examples": [
- "info"
- ]
- },
- "value": {
- "type": "string",
- "description": "The message to communicate.",
- "minLength": 1,
- "examples": [
- "Permission denied, scanning aborted"
- ]
- }
- }
- }
- },
- "analyzer": {
- "type": "object",
- "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the analyzer.",
- "minLength": 1,
- "examples": [
- "gitlab-dast"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the analyzer, not required to be unique.",
- "minLength": 1,
- "examples": [
- "GitLab DAST"
- ]
- },
- "url": {
- "type": "string",
- "format": "uri",
- "pattern": "^https?://.+",
- "description": "A link to more information about the analyzer.",
- "examples": [
- "https://docs.gitlab.com/ee/user/application_security/dast"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the analyzer.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- },
- "version": {
- "type": "string",
- "description": "The version of the analyzer.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- }
- }
- },
- "scanner": {
- "type": "object",
- "description": "Object defining the scanner used to perform the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the scanner.",
- "minLength": 1,
- "examples": [
- "my-sast-scanner"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the scanner, not required to be unique.",
- "minLength": 1,
- "examples": [
- "My SAST Scanner"
- ]
- },
- "url": {
- "type": "string",
- "description": "A link to more information about the scanner.",
- "examples": [
- "https://scanner.url"
- ]
- },
- "version": {
- "type": "string",
- "description": "The version of the scanner.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the scanner.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- }
- }
- },
- "start_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-02-14T16:01:59"
- ]
- },
- "status": {
- "type": "string",
- "description": "Result of the scan.",
- "enum": [
- "success",
- "failure"
- ]
- },
- "type": {
- "type": "string",
- "description": "Type of the scan.",
- "enum": [
- "dast",
- "api_fuzzing"
- ]
- },
- "scanned_resources": {
- "type": "array",
- "description": "The attack surface scanned by DAST.",
- "items": {
- "type": "object",
- "required": [
- "method",
- "url",
- "type"
- ],
- "properties": {
- "method": {
- "type": "string",
- "minLength": 1,
- "description": "HTTP method of the scanned resource.",
- "examples": [
- "GET",
- "POST",
- "HEAD"
- ]
- },
- "url": {
- "type": "string",
- "minLength": 1,
- "description": "URL of the scanned resource.",
- "examples": [
- "http://my.site.com/a-page"
- ]
- },
- "type": {
- "type": "string",
- "minLength": 1,
- "description": "Type of the scanned resource, for DAST, this must be 'url'.",
- "examples": [
- "url"
- ]
- }
- }
- }
- }
- }
- },
- "schema": {
- "type": "string",
- "description": "URI pointing to the validating security report schema.",
- "format": "uri"
- },
- "version": {
- "type": "string",
- "description": "The version of the schema to which the JSON report conforms.",
- "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
- },
- "vulnerabilities": {
- "type": "array",
- "description": "Array of vulnerability objects.",
- "items": {
- "type": "object",
- "description": "Describes the vulnerability using GitLab Flavored Markdown",
- "required": [
- "category",
- "cve",
- "identifiers",
- "location",
- "scanner"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
- "examples": [
- "642735a5-1425-428d-8d4e-3c854885a3c9"
- ]
- },
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
- "name": {
- "type": "string",
- "description": "The name of the vulnerability. This must not include the finding's specific information."
- },
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
- "description": {
- "type": "string",
- "description": "A long text section describing the vulnerability more fully."
- },
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
- "severity": {
- "type": "string",
- "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Info",
- "Unknown",
- "Low",
- "Medium",
- "High",
- "Critical"
- ]
- },
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
- "solution": {
- "type": "string",
- "description": "Explanation of how to fix the vulnerability."
- },
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
- "identifiers": {
- "type": "array",
- "minItems": 1,
- "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
- "items": {
- "type": "object",
- "required": [
- "type",
- "name",
- "value"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
- "minLength": 1
- },
- "name": {
- "type": "string",
- "description": "Human-readable name of the identifier.",
- "minLength": 1
- },
- "url": {
- "type": "string",
- "description": "URL of the identifier's documentation.",
- "format": "uri"
- },
- "value": {
- "type": "string",
- "description": "Value of the identifier, for matching purpose.",
- "minLength": 1
- }
- }
- }
- },
- "links": {
- "type": "array",
- "description": "An array of references to external documentation or articles that describe the vulnerability.",
- "items": {
- "type": "object",
- "required": [
- "url"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the vulnerability details link."
- },
- "url": {
- "type": "string",
- "description": "URL of the vulnerability details document.",
- "format": "uri"
- }
- }
- }
- },
- "details": {
- "$ref": "#/definitions/named_list/properties/items"
- },
- "tracking": {
- "description": "Describes how this vulnerability should be tracked as the project changes.",
- "oneOf": [
- {
- "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
- "required": [
- "items"
- ],
- "properties": {
- "type": {
- "const": "source"
- },
- "items": {
- "type": "array",
- "items": {
- "description": "An item that should be tracked using source-specific tracking methods.",
- "type": "object",
- "required": [
- "signatures"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the file that includes the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the file that includes the vulnerability."
- },
- "signatures": {
- "type": "array",
- "description": "An array of calculated tracking signatures for this tracking item.",
- "minItems": 1,
- "items": {
- "description": "A calculated tracking signature value and metadata.",
- "required": [
- "algorithm",
- "value"
- ],
- "properties": {
- "algorithm": {
- "type": "string",
- "description": "The algorithm used to generate the signature."
- },
- "value": {
- "type": "string",
- "description": "The result of this signature algorithm."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "Each tracking type must declare its own type."
- }
- }
- },
- "flags": {
- "description": "Flags that can be attached to vulnerabilities.",
- "type": "array",
- "items": {
- "type": "object",
- "description": "Informational flags identified and assigned to a vulnerability.",
- "required": [
- "type",
- "origin",
- "description"
- ],
- "properties": {
- "type": {
- "type": "string",
- "minLength": 1,
- "description": "Result of the scan.",
- "enum": [
- "flagged-as-likely-false-positive"
- ]
- },
- "origin": {
- "minLength": 1,
- "description": "Tool that issued the flag.",
- "type": "string"
- },
- "description": {
- "minLength": 1,
- "description": "What the flag is about.",
- "type": "string"
- }
- }
- }
- },
- "evidence": {
- "type": "object",
- "properties": {
- "source": {
- "type": "object",
- "description": "Source of evidence",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "Unique source identifier",
- "examples": [
- "assert:LogAnalysis",
- "assert:StatusCode"
- ]
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Source display name",
- "examples": [
- "Log Analysis",
- "Status Code"
- ]
- },
- "url": {
- "type": "string",
- "description": "Link to additional information",
- "examples": [
- "https://docs.gitlab.com/ee/development/integrations/secure.html"
- ]
- }
- }
- },
- "summary": {
- "type": "string",
- "description": "Human readable string containing evidence of the vulnerability.",
- "examples": [
- "Credit card 4111111111111111 found",
- "Server leaked information nginx/1.17.6"
- ]
- },
- "request": {
- "type": "object",
- "description": "An HTTP request.",
- "required": [
- "headers",
- "method",
- "url"
- ],
- "properties": {
- "headers": {
- "type": "array",
- "description": "HTTP headers present on the request.",
- "items": {
- "type": "object",
- "required": [
- "name",
- "value"
- ],
- "properties": {
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Name of the HTTP header.",
- "examples": [
- "Accept",
- "Content-Length",
- "Content-Type"
- ]
- },
- "value": {
- "type": "string",
- "minLength": 1,
- "description": "Value of the HTTP header.",
- "examples": [
- "*/*",
- "560",
- "application/json; charset=utf-8"
- ]
- }
- }
- }
- },
- "method": {
- "type": "string",
- "minLength": 1,
- "description": "HTTP method used in the request.",
- "examples": [
- "GET",
- "POST"
- ]
- },
- "url": {
- "type": "string",
- "minLength": 1,
- "description": "URL of the request.",
- "examples": [
- "http://my.site.com/vulnerable-endpoint?show-credit-card"
- ]
- },
- "body": {
- "type": "string",
- "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.",
- "examples": [
- "user=jsmith&first=%27&last=smith"
- ]
- }
- }
- },
- "response": {
- "type": "object",
- "description": "An HTTP response.",
- "required": [
- "headers",
- "reason_phrase",
- "status_code"
- ],
- "properties": {
- "headers": {
- "type": "array",
- "description": "HTTP headers present on the request.",
- "items": {
- "type": "object",
- "required": [
- "name",
- "value"
- ],
- "properties": {
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Name of the HTTP header.",
- "examples": [
- "Accept",
- "Content-Length",
- "Content-Type"
- ]
- },
- "value": {
- "type": "string",
- "minLength": 1,
- "description": "Value of the HTTP header.",
- "examples": [
- "*/*",
- "560",
- "application/json; charset=utf-8"
- ]
- }
- }
- }
- },
- "reason_phrase": {
- "type": "string",
- "description": "HTTP reason phrase of the response.",
- "examples": [
- "OK",
- "Internal Server Error"
- ]
- },
- "status_code": {
- "type": "integer",
- "description": "HTTP status code of the response.",
- "examples": [
- 200,
- 500
- ]
- },
- "body": {
- "type": "string",
- "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.",
- "examples": [
- "{\"user_id\": 2}"
- ]
- }
- }
- },
- "supporting_messages": {
- "type": "array",
- "description": "Array of supporting http messages.",
- "items": {
- "type": "object",
- "description": "A supporting http message.",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Message display name.",
- "examples": [
- "Unmodified",
- "Recorded"
- ]
- },
- "request": {
- "type": "object",
- "description": "An HTTP request.",
- "required": [
- "headers",
- "method",
- "url"
- ],
- "properties": {
- "headers": {
- "type": "array",
- "description": "HTTP headers present on the request.",
- "items": {
- "type": "object",
- "required": [
- "name",
- "value"
- ],
- "properties": {
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Name of the HTTP header.",
- "examples": [
- "Accept",
- "Content-Length",
- "Content-Type"
- ]
- },
- "value": {
- "type": "string",
- "minLength": 1,
- "description": "Value of the HTTP header.",
- "examples": [
- "*/*",
- "560",
- "application/json; charset=utf-8"
- ]
- }
- }
- }
- },
- "method": {
- "type": "string",
- "minLength": 1,
- "description": "HTTP method used in the request.",
- "examples": [
- "GET",
- "POST"
- ]
- },
- "url": {
- "type": "string",
- "minLength": 1,
- "description": "URL of the request.",
- "examples": [
- "http://my.site.com/vulnerable-endpoint?show-credit-card"
- ]
- },
- "body": {
- "type": "string",
- "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.",
- "examples": [
- "user=jsmith&first=%27&last=smith"
- ]
- }
- }
- },
- "response": {
- "type": "object",
- "description": "An HTTP response.",
- "required": [
- "headers",
- "reason_phrase",
- "status_code"
- ],
- "properties": {
- "headers": {
- "type": "array",
- "description": "HTTP headers present on the request.",
- "items": {
- "type": "object",
- "required": [
- "name",
- "value"
- ],
- "properties": {
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Name of the HTTP header.",
- "examples": [
- "Accept",
- "Content-Length",
- "Content-Type"
- ]
- },
- "value": {
- "type": "string",
- "minLength": 1,
- "description": "Value of the HTTP header.",
- "examples": [
- "*/*",
- "560",
- "application/json; charset=utf-8"
- ]
- }
- }
- }
- },
- "reason_phrase": {
- "type": "string",
- "description": "HTTP reason phrase of the response.",
- "examples": [
- "OK",
- "Internal Server Error"
- ]
- },
- "status_code": {
- "type": "integer",
- "description": "HTTP status code of the response.",
- "examples": [
- 200,
- 500
- ]
- },
- "body": {
- "type": "string",
- "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.",
- "examples": [
- "{\"user_id\": 2}"
- ]
- }
- }
- }
- }
- }
- }
- }
- },
- "location": {
- "type": "object",
- "description": "Identifies the vulnerability's location.",
- "properties": {
- "hostname": {
- "type": "string",
- "description": "The protocol, domain, and port of the application where the vulnerability was found."
- },
- "method": {
- "type": "string",
- "description": "The HTTP method that was used to request the URL where the vulnerability was found."
- },
- "param": {
- "type": "string",
- "description": "A value provided by a vulnerability rule related to the found vulnerability. Examples include a header value, or a parameter used in a HTTP POST."
- },
- "path": {
- "type": "string",
- "description": "The path of the URL where the vulnerability was found. Typically, this would start with a forward slash."
- }
- }
- },
- "assets": {
- "type": "array",
- "description": "Array of build assets associated with vulnerability.",
- "items": {
- "type": "object",
- "description": "Describes an asset associated with vulnerability.",
- "required": [
- "type",
- "name",
- "url"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "The type of asset",
- "enum": [
- "http_session",
- "postman"
- ]
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Display name for asset",
- "examples": [
- "HTTP Messages",
- "Postman Collection"
- ]
- },
- "url": {
- "type": "string",
- "minLength": 1,
- "description": "Link to asset in build artifacts",
- "examples": [
- "https://gitlab.com/gitlab-org/security-products/dast/-/jobs/626397001/artifacts/file//output/zap_session.data"
- ]
- }
- }
- }
- },
- "discovered_at": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss.sss, representing when the vulnerability was discovered",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}\\.\\d{3}$",
- "examples": [
- "2020-01-28T03:26:02.956"
- ]
- }
- }
- }
- },
- "remediations": {
- "type": "array",
- "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
- "items": {
- "type": "object",
- "required": [
- "fixes",
- "summary",
- "diff"
- ],
- "properties": {
- "fixes": {
- "type": "array",
- "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
- "items": {
- "type": "object",
- "required": [
- "cve"
- ],
- "properties": {
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- }
- }
- }
- },
- "summary": {
- "type": "string",
- "minLength": 1,
- "description": "An overview of how the vulnerabilities were fixed."
- },
- "diff": {
- "type": "string",
- "minLength": 1,
- "description": "A base64-encoded remediation code diff, compatible with git apply."
- }
- }
- }
- }
- }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/dependency-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/dependency-scanning-report-format.json
deleted file mode 100644
index 1e3f4188845..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/dependency-scanning-report-format.json
+++ /dev/null
@@ -1,968 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Report format for GitLab Dependency Scanning",
- "description": "This schema provides the the report format for Dependency Scanning analyzers (https://docs.gitlab.com/ee/user/application_security/dependency_scanning).",
- "definitions": {
- "detail_type": {
- "oneOf": [
- {
- "$ref": "#/definitions/named_list"
- },
- {
- "$ref": "#/definitions/list"
- },
- {
- "$ref": "#/definitions/table"
- },
- {
- "$ref": "#/definitions/text"
- },
- {
- "$ref": "#/definitions/url"
- },
- {
- "$ref": "#/definitions/code"
- },
- {
- "$ref": "#/definitions/value"
- },
- {
- "$ref": "#/definitions/diff"
- },
- {
- "$ref": "#/definitions/markdown"
- },
- {
- "$ref": "#/definitions/commit"
- },
- {
- "$ref": "#/definitions/file_location"
- },
- {
- "$ref": "#/definitions/module_location"
- }
- ]
- },
- "text_value": {
- "type": "string"
- },
- "named_field": {
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "$ref": "#/definitions/text_value",
- "minLength": 1
- },
- "description": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "named_list": {
- "type": "object",
- "description": "An object with named and typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "named-list"
- },
- "items": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- {
- "$ref": "#/definitions/named_field"
- },
- {
- "$ref": "#/definitions/detail_type"
- }
- ]
- }
- }
- }
- }
- },
- "list": {
- "type": "object",
- "description": "A list of typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- },
- "table": {
- "type": "object",
- "description": "A table of typed fields",
- "required": [
- "type",
- "rows"
- ],
- "properties": {
- "type": {
- "const": "table"
- },
- "header": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- },
- "rows": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- }
- },
- "text": {
- "type": "object",
- "description": "Raw text",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "text"
- },
- "value": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "url": {
- "type": "object",
- "description": "A single URL",
- "required": [
- "type",
- "href"
- ],
- "properties": {
- "type": {
- "const": "url"
- },
- "text": {
- "$ref": "#/definitions/text_value"
- },
- "href": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "http://mysite.com"
- ]
- }
- }
- },
- "code": {
- "type": "object",
- "description": "A codeblock",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "code"
- },
- "value": {
- "type": "string"
- },
- "lang": {
- "type": "string",
- "description": "A programming language"
- }
- }
- },
- "value": {
- "type": "object",
- "description": "A field that can store a range of types of value",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "value"
- },
- "value": {
- "type": [
- "number",
- "string",
- "boolean"
- ]
- }
- }
- },
- "diff": {
- "type": "object",
- "description": "A diff",
- "required": [
- "type",
- "before",
- "after"
- ],
- "properties": {
- "type": {
- "const": "diff"
- },
- "before": {
- "type": "string"
- },
- "after": {
- "type": "string"
- }
- }
- },
- "markdown": {
- "type": "object",
- "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "markdown"
- },
- "value": {
- "$ref": "#/definitions/text_value",
- "examples": [
- "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
- ]
- }
- }
- },
- "commit": {
- "type": "object",
- "description": "A commit/tag/branch within the GitLab project",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "commit"
- },
- "value": {
- "type": "string",
- "description": "The commit SHA",
- "minLength": 1
- }
- }
- },
- "file_location": {
- "type": "object",
- "description": "A location within a file in the project",
- "required": [
- "type",
- "file_name",
- "line_start"
- ],
- "properties": {
- "type": {
- "const": "file-location"
- },
- "file_name": {
- "type": "string",
- "minLength": 1
- },
- "line_start": {
- "type": "integer"
- },
- "line_end": {
- "type": "integer"
- }
- }
- },
- "module_location": {
- "type": "object",
- "description": "A location within a binary module of the form module+relative_offset",
- "required": [
- "type",
- "module_name",
- "offset"
- ],
- "properties": {
- "type": {
- "const": "module-location"
- },
- "module_name": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "compiled_binary"
- ]
- },
- "offset": {
- "type": "integer",
- "examples": [
- 100
- ]
- }
- }
- }
- },
- "self": {
- "version": "14.1.1"
- },
- "required": [
- "dependency_files",
- "version",
- "vulnerabilities"
- ],
- "additionalProperties": true,
- "properties": {
- "scan": {
- "type": "object",
- "required": [
- "end_time",
- "scanner",
- "start_time",
- "status",
- "type"
- ],
- "properties": {
- "end_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-01-28T03:26:02"
- ]
- },
- "messages": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Communication intended for the initiator of a scan.",
- "required": [
- "level",
- "value"
- ],
- "properties": {
- "level": {
- "type": "string",
- "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
- "enum": [
- "info",
- "warn",
- "fatal"
- ],
- "examples": [
- "info"
- ]
- },
- "value": {
- "type": "string",
- "description": "The message to communicate.",
- "minLength": 1,
- "examples": [
- "Permission denied, scanning aborted"
- ]
- }
- }
- }
- },
- "analyzer": {
- "type": "object",
- "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the analyzer.",
- "minLength": 1,
- "examples": [
- "gitlab-dast"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the analyzer, not required to be unique.",
- "minLength": 1,
- "examples": [
- "GitLab DAST"
- ]
- },
- "url": {
- "type": "string",
- "format": "uri",
- "pattern": "^https?://.+",
- "description": "A link to more information about the analyzer.",
- "examples": [
- "https://docs.gitlab.com/ee/user/application_security/dast"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the analyzer.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- },
- "version": {
- "type": "string",
- "description": "The version of the analyzer.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- }
- }
- },
- "scanner": {
- "type": "object",
- "description": "Object defining the scanner used to perform the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the scanner.",
- "minLength": 1,
- "examples": [
- "my-sast-scanner"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the scanner, not required to be unique.",
- "minLength": 1,
- "examples": [
- "My SAST Scanner"
- ]
- },
- "url": {
- "type": "string",
- "description": "A link to more information about the scanner.",
- "examples": [
- "https://scanner.url"
- ]
- },
- "version": {
- "type": "string",
- "description": "The version of the scanner.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the scanner.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- }
- }
- },
- "start_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-02-14T16:01:59"
- ]
- },
- "status": {
- "type": "string",
- "description": "Result of the scan.",
- "enum": [
- "success",
- "failure"
- ]
- },
- "type": {
- "type": "string",
- "description": "Type of the scan.",
- "enum": [
- "dependency_scanning"
- ]
- }
- }
- },
- "schema": {
- "type": "string",
- "description": "URI pointing to the validating security report schema.",
- "format": "uri"
- },
- "version": {
- "type": "string",
- "description": "The version of the schema to which the JSON report conforms.",
- "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
- },
- "vulnerabilities": {
- "type": "array",
- "description": "Array of vulnerability objects.",
- "items": {
- "type": "object",
- "description": "Describes the vulnerability using GitLab Flavored Markdown",
- "required": [
- "category",
- "cve",
- "identifiers",
- "location",
- "scanner"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
- "examples": [
- "642735a5-1425-428d-8d4e-3c854885a3c9"
- ]
- },
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
- "name": {
- "type": "string",
- "description": "The name of the vulnerability. This must not include the finding's specific information."
- },
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
- "description": {
- "type": "string",
- "description": "A long text section describing the vulnerability more fully."
- },
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
- "severity": {
- "type": "string",
- "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Info",
- "Unknown",
- "Low",
- "Medium",
- "High",
- "Critical"
- ]
- },
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
- "solution": {
- "type": "string",
- "description": "Explanation of how to fix the vulnerability."
- },
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
- "identifiers": {
- "type": "array",
- "minItems": 1,
- "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
- "items": {
- "type": "object",
- "required": [
- "type",
- "name",
- "value"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
- "minLength": 1
- },
- "name": {
- "type": "string",
- "description": "Human-readable name of the identifier.",
- "minLength": 1
- },
- "url": {
- "type": "string",
- "description": "URL of the identifier's documentation.",
- "format": "uri"
- },
- "value": {
- "type": "string",
- "description": "Value of the identifier, for matching purpose.",
- "minLength": 1
- }
- }
- }
- },
- "links": {
- "type": "array",
- "description": "An array of references to external documentation or articles that describe the vulnerability.",
- "items": {
- "type": "object",
- "required": [
- "url"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the vulnerability details link."
- },
- "url": {
- "type": "string",
- "description": "URL of the vulnerability details document.",
- "format": "uri"
- }
- }
- }
- },
- "details": {
- "$ref": "#/definitions/named_list/properties/items"
- },
- "tracking": {
- "description": "Describes how this vulnerability should be tracked as the project changes.",
- "oneOf": [
- {
- "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
- "required": [
- "items"
- ],
- "properties": {
- "type": {
- "const": "source"
- },
- "items": {
- "type": "array",
- "items": {
- "description": "An item that should be tracked using source-specific tracking methods.",
- "type": "object",
- "required": [
- "signatures"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the file that includes the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the file that includes the vulnerability."
- },
- "signatures": {
- "type": "array",
- "description": "An array of calculated tracking signatures for this tracking item.",
- "minItems": 1,
- "items": {
- "description": "A calculated tracking signature value and metadata.",
- "required": [
- "algorithm",
- "value"
- ],
- "properties": {
- "algorithm": {
- "type": "string",
- "description": "The algorithm used to generate the signature."
- },
- "value": {
- "type": "string",
- "description": "The result of this signature algorithm."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "Each tracking type must declare its own type."
- }
- }
- },
- "flags": {
- "description": "Flags that can be attached to vulnerabilities.",
- "type": "array",
- "items": {
- "type": "object",
- "description": "Informational flags identified and assigned to a vulnerability.",
- "required": [
- "type",
- "origin",
- "description"
- ],
- "properties": {
- "type": {
- "type": "string",
- "minLength": 1,
- "description": "Result of the scan.",
- "enum": [
- "flagged-as-likely-false-positive"
- ]
- },
- "origin": {
- "minLength": 1,
- "description": "Tool that issued the flag.",
- "type": "string"
- },
- "description": {
- "minLength": 1,
- "description": "What the flag is about.",
- "type": "string"
- }
- }
- }
- },
- "location": {
- "type": "object",
- "description": "Identifies the vulnerability's location.",
- "required": [
- "file",
- "dependency"
- ],
- "properties": {
- "file": {
- "type": "string",
- "minLength": 1,
- "description": "Path to the manifest or lock file where the dependency is declared (such as yarn.lock)."
- },
- "dependency": {
- "type": "object",
- "description": "Describes the dependency of a project where the vulnerability is located.",
- "properties": {
- "package": {
- "type": "object",
- "description": "Provides information on the package where the vulnerability is located.",
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the package where the vulnerability is located."
- }
- }
- },
- "version": {
- "type": "string",
- "description": "Version of the vulnerable package."
- },
- "iid": {
- "description": "ID that identifies the dependency in the scope of a dependency file.",
- "type": "number"
- },
- "direct": {
- "type": "boolean",
- "description": "Tells whether this is a direct, top-level dependency of the scanned project."
- },
- "dependency_path": {
- "type": "array",
- "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.",
- "items": {
- "type": "object",
- "required": [
- "iid"
- ],
- "properties": {
- "iid": {
- "type": "number",
- "description": "ID that is unique in the scope of a parent object, and specific to the resource type."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- }
- },
- "remediations": {
- "type": "array",
- "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
- "items": {
- "type": "object",
- "required": [
- "fixes",
- "summary",
- "diff"
- ],
- "properties": {
- "fixes": {
- "type": "array",
- "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
- "items": {
- "type": "object",
- "required": [
- "cve"
- ],
- "properties": {
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- }
- }
- }
- },
- "summary": {
- "type": "string",
- "minLength": 1,
- "description": "An overview of how the vulnerabilities were fixed."
- },
- "diff": {
- "type": "string",
- "minLength": 1,
- "description": "A base64-encoded remediation code diff, compatible with git apply."
- }
- }
- }
- },
- "dependency_files": {
- "type": "array",
- "description": "List of dependency files identified in the project.",
- "items": {
- "type": "object",
- "required": [
- "path",
- "package_manager",
- "dependencies"
- ],
- "properties": {
- "path": {
- "type": "string",
- "minLength": 1
- },
- "package_manager": {
- "type": "string",
- "minLength": 1
- },
- "dependencies": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Describes the dependency of a project where the vulnerability is located.",
- "properties": {
- "package": {
- "type": "object",
- "description": "Provides information on the package where the vulnerability is located.",
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the package where the vulnerability is located."
- }
- }
- },
- "version": {
- "type": "string",
- "description": "Version of the vulnerable package."
- },
- "iid": {
- "description": "ID that identifies the dependency in the scope of a dependency file.",
- "type": "number"
- },
- "direct": {
- "type": "boolean",
- "description": "Tells whether this is a direct, top-level dependency of the scanned project."
- },
- "dependency_path": {
- "type": "array",
- "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.",
- "items": {
- "type": "object",
- "required": [
- "iid"
- ],
- "properties": {
- "iid": {
- "type": "number",
- "description": "ID that is unique in the scope of a parent object, and specific to the resource type."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- }
- }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/sast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/sast-report-format.json
deleted file mode 100644
index 4c57d20dbaa..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/sast-report-format.json
+++ /dev/null
@@ -1,869 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Report format for GitLab SAST",
- "description": "This schema provides the report format for Static Application Security Testing analyzers (https://docs.gitlab.com/ee/user/application_security/sast).",
- "definitions": {
- "detail_type": {
- "oneOf": [
- {
- "$ref": "#/definitions/named_list"
- },
- {
- "$ref": "#/definitions/list"
- },
- {
- "$ref": "#/definitions/table"
- },
- {
- "$ref": "#/definitions/text"
- },
- {
- "$ref": "#/definitions/url"
- },
- {
- "$ref": "#/definitions/code"
- },
- {
- "$ref": "#/definitions/value"
- },
- {
- "$ref": "#/definitions/diff"
- },
- {
- "$ref": "#/definitions/markdown"
- },
- {
- "$ref": "#/definitions/commit"
- },
- {
- "$ref": "#/definitions/file_location"
- },
- {
- "$ref": "#/definitions/module_location"
- }
- ]
- },
- "text_value": {
- "type": "string"
- },
- "named_field": {
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "$ref": "#/definitions/text_value",
- "minLength": 1
- },
- "description": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "named_list": {
- "type": "object",
- "description": "An object with named and typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "named-list"
- },
- "items": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- {
- "$ref": "#/definitions/named_field"
- },
- {
- "$ref": "#/definitions/detail_type"
- }
- ]
- }
- }
- }
- }
- },
- "list": {
- "type": "object",
- "description": "A list of typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- },
- "table": {
- "type": "object",
- "description": "A table of typed fields",
- "required": [
- "type",
- "rows"
- ],
- "properties": {
- "type": {
- "const": "table"
- },
- "header": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- },
- "rows": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- }
- },
- "text": {
- "type": "object",
- "description": "Raw text",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "text"
- },
- "value": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "url": {
- "type": "object",
- "description": "A single URL",
- "required": [
- "type",
- "href"
- ],
- "properties": {
- "type": {
- "const": "url"
- },
- "text": {
- "$ref": "#/definitions/text_value"
- },
- "href": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "http://mysite.com"
- ]
- }
- }
- },
- "code": {
- "type": "object",
- "description": "A codeblock",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "code"
- },
- "value": {
- "type": "string"
- },
- "lang": {
- "type": "string",
- "description": "A programming language"
- }
- }
- },
- "value": {
- "type": "object",
- "description": "A field that can store a range of types of value",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "value"
- },
- "value": {
- "type": [
- "number",
- "string",
- "boolean"
- ]
- }
- }
- },
- "diff": {
- "type": "object",
- "description": "A diff",
- "required": [
- "type",
- "before",
- "after"
- ],
- "properties": {
- "type": {
- "const": "diff"
- },
- "before": {
- "type": "string"
- },
- "after": {
- "type": "string"
- }
- }
- },
- "markdown": {
- "type": "object",
- "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "markdown"
- },
- "value": {
- "$ref": "#/definitions/text_value",
- "examples": [
- "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
- ]
- }
- }
- },
- "commit": {
- "type": "object",
- "description": "A commit/tag/branch within the GitLab project",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "commit"
- },
- "value": {
- "type": "string",
- "description": "The commit SHA",
- "minLength": 1
- }
- }
- },
- "file_location": {
- "type": "object",
- "description": "A location within a file in the project",
- "required": [
- "type",
- "file_name",
- "line_start"
- ],
- "properties": {
- "type": {
- "const": "file-location"
- },
- "file_name": {
- "type": "string",
- "minLength": 1
- },
- "line_start": {
- "type": "integer"
- },
- "line_end": {
- "type": "integer"
- }
- }
- },
- "module_location": {
- "type": "object",
- "description": "A location within a binary module of the form module+relative_offset",
- "required": [
- "type",
- "module_name",
- "offset"
- ],
- "properties": {
- "type": {
- "const": "module-location"
- },
- "module_name": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "compiled_binary"
- ]
- },
- "offset": {
- "type": "integer",
- "examples": [
- 100
- ]
- }
- }
- }
- },
- "self": {
- "version": "14.1.1"
- },
- "required": [
- "version",
- "vulnerabilities"
- ],
- "additionalProperties": true,
- "properties": {
- "scan": {
- "type": "object",
- "required": [
- "end_time",
- "scanner",
- "start_time",
- "status",
- "type"
- ],
- "properties": {
- "end_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-01-28T03:26:02"
- ]
- },
- "messages": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Communication intended for the initiator of a scan.",
- "required": [
- "level",
- "value"
- ],
- "properties": {
- "level": {
- "type": "string",
- "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
- "enum": [
- "info",
- "warn",
- "fatal"
- ],
- "examples": [
- "info"
- ]
- },
- "value": {
- "type": "string",
- "description": "The message to communicate.",
- "minLength": 1,
- "examples": [
- "Permission denied, scanning aborted"
- ]
- }
- }
- }
- },
- "analyzer": {
- "type": "object",
- "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the analyzer.",
- "minLength": 1,
- "examples": [
- "gitlab-dast"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the analyzer, not required to be unique.",
- "minLength": 1,
- "examples": [
- "GitLab DAST"
- ]
- },
- "url": {
- "type": "string",
- "format": "uri",
- "pattern": "^https?://.+",
- "description": "A link to more information about the analyzer.",
- "examples": [
- "https://docs.gitlab.com/ee/user/application_security/dast"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the analyzer.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- },
- "version": {
- "type": "string",
- "description": "The version of the analyzer.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- }
- }
- },
- "scanner": {
- "type": "object",
- "description": "Object defining the scanner used to perform the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the scanner.",
- "minLength": 1,
- "examples": [
- "my-sast-scanner"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the scanner, not required to be unique.",
- "minLength": 1,
- "examples": [
- "My SAST Scanner"
- ]
- },
- "url": {
- "type": "string",
- "description": "A link to more information about the scanner.",
- "examples": [
- "https://scanner.url"
- ]
- },
- "version": {
- "type": "string",
- "description": "The version of the scanner.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the scanner.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- }
- }
- },
- "start_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-02-14T16:01:59"
- ]
- },
- "status": {
- "type": "string",
- "description": "Result of the scan.",
- "enum": [
- "success",
- "failure"
- ]
- },
- "type": {
- "type": "string",
- "description": "Type of the scan.",
- "enum": [
- "sast"
- ]
- }
- }
- },
- "schema": {
- "type": "string",
- "description": "URI pointing to the validating security report schema.",
- "format": "uri"
- },
- "version": {
- "type": "string",
- "description": "The version of the schema to which the JSON report conforms.",
- "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
- },
- "vulnerabilities": {
- "type": "array",
- "description": "Array of vulnerability objects.",
- "items": {
- "type": "object",
- "description": "Describes the vulnerability using GitLab Flavored Markdown",
- "required": [
- "category",
- "cve",
- "identifiers",
- "location",
- "scanner"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
- "examples": [
- "642735a5-1425-428d-8d4e-3c854885a3c9"
- ]
- },
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
- "name": {
- "type": "string",
- "description": "The name of the vulnerability. This must not include the finding's specific information."
- },
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
- "description": {
- "type": "string",
- "description": "A long text section describing the vulnerability more fully."
- },
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
- "severity": {
- "type": "string",
- "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Info",
- "Unknown",
- "Low",
- "Medium",
- "High",
- "Critical"
- ]
- },
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
- "solution": {
- "type": "string",
- "description": "Explanation of how to fix the vulnerability."
- },
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
- "identifiers": {
- "type": "array",
- "minItems": 1,
- "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
- "items": {
- "type": "object",
- "required": [
- "type",
- "name",
- "value"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
- "minLength": 1
- },
- "name": {
- "type": "string",
- "description": "Human-readable name of the identifier.",
- "minLength": 1
- },
- "url": {
- "type": "string",
- "description": "URL of the identifier's documentation.",
- "format": "uri"
- },
- "value": {
- "type": "string",
- "description": "Value of the identifier, for matching purpose.",
- "minLength": 1
- }
- }
- }
- },
- "links": {
- "type": "array",
- "description": "An array of references to external documentation or articles that describe the vulnerability.",
- "items": {
- "type": "object",
- "required": [
- "url"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the vulnerability details link."
- },
- "url": {
- "type": "string",
- "description": "URL of the vulnerability details document.",
- "format": "uri"
- }
- }
- }
- },
- "details": {
- "$ref": "#/definitions/named_list/properties/items"
- },
- "tracking": {
- "description": "Describes how this vulnerability should be tracked as the project changes.",
- "oneOf": [
- {
- "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
- "required": [
- "items"
- ],
- "properties": {
- "type": {
- "const": "source"
- },
- "items": {
- "type": "array",
- "items": {
- "description": "An item that should be tracked using source-specific tracking methods.",
- "type": "object",
- "required": [
- "signatures"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the file that includes the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the file that includes the vulnerability."
- },
- "signatures": {
- "type": "array",
- "description": "An array of calculated tracking signatures for this tracking item.",
- "minItems": 1,
- "items": {
- "description": "A calculated tracking signature value and metadata.",
- "required": [
- "algorithm",
- "value"
- ],
- "properties": {
- "algorithm": {
- "type": "string",
- "description": "The algorithm used to generate the signature."
- },
- "value": {
- "type": "string",
- "description": "The result of this signature algorithm."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "Each tracking type must declare its own type."
- }
- }
- },
- "flags": {
- "description": "Flags that can be attached to vulnerabilities.",
- "type": "array",
- "items": {
- "type": "object",
- "description": "Informational flags identified and assigned to a vulnerability.",
- "required": [
- "type",
- "origin",
- "description"
- ],
- "properties": {
- "type": {
- "type": "string",
- "minLength": 1,
- "description": "Result of the scan.",
- "enum": [
- "flagged-as-likely-false-positive"
- ]
- },
- "origin": {
- "minLength": 1,
- "description": "Tool that issued the flag.",
- "type": "string"
- },
- "description": {
- "minLength": 1,
- "description": "What the flag is about.",
- "type": "string"
- }
- }
- }
- },
- "location": {
- "type": "object",
- "description": "Identifies the vulnerability's location.",
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the code affected by the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the code affected by the vulnerability."
- },
- "class": {
- "type": "string",
- "description": "Provides the name of the class where the vulnerability is located."
- },
- "method": {
- "type": "string",
- "description": "Provides the name of the method where the vulnerability is located."
- }
- }
- },
- "raw_source_code_extract": {
- "type": "string",
- "description": "Provides an unsanitized excerpt of the affected source code."
- }
- }
- }
- },
- "remediations": {
- "type": "array",
- "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
- "items": {
- "type": "object",
- "required": [
- "fixes",
- "summary",
- "diff"
- ],
- "properties": {
- "fixes": {
- "type": "array",
- "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
- "items": {
- "type": "object",
- "required": [
- "cve"
- ],
- "properties": {
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- }
- }
- }
- },
- "summary": {
- "type": "string",
- "minLength": 1,
- "description": "An overview of how the vulnerabilities were fixed."
- },
- "diff": {
- "type": "string",
- "minLength": 1,
- "description": "A base64-encoded remediation code diff, compatible with git apply."
- }
- }
- }
- }
- }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/secret-detection-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/secret-detection-report-format.json
deleted file mode 100644
index b1337954e97..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/secret-detection-report-format.json
+++ /dev/null
@@ -1,892 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Report format for GitLab Secret Detection",
- "description": "This schema provides the the report format for the Secret Detection analyzer (https://docs.gitlab.com/ee/user/application_security/secret_detection)",
- "definitions": {
- "detail_type": {
- "oneOf": [
- {
- "$ref": "#/definitions/named_list"
- },
- {
- "$ref": "#/definitions/list"
- },
- {
- "$ref": "#/definitions/table"
- },
- {
- "$ref": "#/definitions/text"
- },
- {
- "$ref": "#/definitions/url"
- },
- {
- "$ref": "#/definitions/code"
- },
- {
- "$ref": "#/definitions/value"
- },
- {
- "$ref": "#/definitions/diff"
- },
- {
- "$ref": "#/definitions/markdown"
- },
- {
- "$ref": "#/definitions/commit"
- },
- {
- "$ref": "#/definitions/file_location"
- },
- {
- "$ref": "#/definitions/module_location"
- }
- ]
- },
- "text_value": {
- "type": "string"
- },
- "named_field": {
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "$ref": "#/definitions/text_value",
- "minLength": 1
- },
- "description": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "named_list": {
- "type": "object",
- "description": "An object with named and typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "named-list"
- },
- "items": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- {
- "$ref": "#/definitions/named_field"
- },
- {
- "$ref": "#/definitions/detail_type"
- }
- ]
- }
- }
- }
- }
- },
- "list": {
- "type": "object",
- "description": "A list of typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- },
- "table": {
- "type": "object",
- "description": "A table of typed fields",
- "required": [
- "type",
- "rows"
- ],
- "properties": {
- "type": {
- "const": "table"
- },
- "header": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- },
- "rows": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- }
- },
- "text": {
- "type": "object",
- "description": "Raw text",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "text"
- },
- "value": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "url": {
- "type": "object",
- "description": "A single URL",
- "required": [
- "type",
- "href"
- ],
- "properties": {
- "type": {
- "const": "url"
- },
- "text": {
- "$ref": "#/definitions/text_value"
- },
- "href": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "http://mysite.com"
- ]
- }
- }
- },
- "code": {
- "type": "object",
- "description": "A codeblock",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "code"
- },
- "value": {
- "type": "string"
- },
- "lang": {
- "type": "string",
- "description": "A programming language"
- }
- }
- },
- "value": {
- "type": "object",
- "description": "A field that can store a range of types of value",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "value"
- },
- "value": {
- "type": [
- "number",
- "string",
- "boolean"
- ]
- }
- }
- },
- "diff": {
- "type": "object",
- "description": "A diff",
- "required": [
- "type",
- "before",
- "after"
- ],
- "properties": {
- "type": {
- "const": "diff"
- },
- "before": {
- "type": "string"
- },
- "after": {
- "type": "string"
- }
- }
- },
- "markdown": {
- "type": "object",
- "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "markdown"
- },
- "value": {
- "$ref": "#/definitions/text_value",
- "examples": [
- "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
- ]
- }
- }
- },
- "commit": {
- "type": "object",
- "description": "A commit/tag/branch within the GitLab project",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "commit"
- },
- "value": {
- "type": "string",
- "description": "The commit SHA",
- "minLength": 1
- }
- }
- },
- "file_location": {
- "type": "object",
- "description": "A location within a file in the project",
- "required": [
- "type",
- "file_name",
- "line_start"
- ],
- "properties": {
- "type": {
- "const": "file-location"
- },
- "file_name": {
- "type": "string",
- "minLength": 1
- },
- "line_start": {
- "type": "integer"
- },
- "line_end": {
- "type": "integer"
- }
- }
- },
- "module_location": {
- "type": "object",
- "description": "A location within a binary module of the form module+relative_offset",
- "required": [
- "type",
- "module_name",
- "offset"
- ],
- "properties": {
- "type": {
- "const": "module-location"
- },
- "module_name": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "compiled_binary"
- ]
- },
- "offset": {
- "type": "integer",
- "examples": [
- 100
- ]
- }
- }
- }
- },
- "self": {
- "version": "14.1.1"
- },
- "required": [
- "version",
- "vulnerabilities"
- ],
- "additionalProperties": true,
- "properties": {
- "scan": {
- "type": "object",
- "required": [
- "end_time",
- "scanner",
- "start_time",
- "status",
- "type"
- ],
- "properties": {
- "end_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-01-28T03:26:02"
- ]
- },
- "messages": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Communication intended for the initiator of a scan.",
- "required": [
- "level",
- "value"
- ],
- "properties": {
- "level": {
- "type": "string",
- "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
- "enum": [
- "info",
- "warn",
- "fatal"
- ],
- "examples": [
- "info"
- ]
- },
- "value": {
- "type": "string",
- "description": "The message to communicate.",
- "minLength": 1,
- "examples": [
- "Permission denied, scanning aborted"
- ]
- }
- }
- }
- },
- "analyzer": {
- "type": "object",
- "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the analyzer.",
- "minLength": 1,
- "examples": [
- "gitlab-dast"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the analyzer, not required to be unique.",
- "minLength": 1,
- "examples": [
- "GitLab DAST"
- ]
- },
- "url": {
- "type": "string",
- "format": "uri",
- "pattern": "^https?://.+",
- "description": "A link to more information about the analyzer.",
- "examples": [
- "https://docs.gitlab.com/ee/user/application_security/dast"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the analyzer.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- },
- "version": {
- "type": "string",
- "description": "The version of the analyzer.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- }
- }
- },
- "scanner": {
- "type": "object",
- "description": "Object defining the scanner used to perform the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the scanner.",
- "minLength": 1,
- "examples": [
- "my-sast-scanner"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the scanner, not required to be unique.",
- "minLength": 1,
- "examples": [
- "My SAST Scanner"
- ]
- },
- "url": {
- "type": "string",
- "description": "A link to more information about the scanner.",
- "examples": [
- "https://scanner.url"
- ]
- },
- "version": {
- "type": "string",
- "description": "The version of the scanner.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the scanner.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- }
- }
- },
- "start_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-02-14T16:01:59"
- ]
- },
- "status": {
- "type": "string",
- "description": "Result of the scan.",
- "enum": [
- "success",
- "failure"
- ]
- },
- "type": {
- "type": "string",
- "description": "Type of the scan.",
- "enum": [
- "secret_detection"
- ]
- }
- }
- },
- "schema": {
- "type": "string",
- "description": "URI pointing to the validating security report schema.",
- "format": "uri"
- },
- "version": {
- "type": "string",
- "description": "The version of the schema to which the JSON report conforms.",
- "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
- },
- "vulnerabilities": {
- "type": "array",
- "description": "Array of vulnerability objects.",
- "items": {
- "type": "object",
- "description": "Describes the vulnerability using GitLab Flavored Markdown",
- "required": [
- "category",
- "cve",
- "identifiers",
- "location",
- "scanner"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
- "examples": [
- "642735a5-1425-428d-8d4e-3c854885a3c9"
- ]
- },
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
- "name": {
- "type": "string",
- "description": "The name of the vulnerability. This must not include the finding's specific information."
- },
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
- "description": {
- "type": "string",
- "description": "A long text section describing the vulnerability more fully."
- },
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
- "severity": {
- "type": "string",
- "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Info",
- "Unknown",
- "Low",
- "Medium",
- "High",
- "Critical"
- ]
- },
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
- "solution": {
- "type": "string",
- "description": "Explanation of how to fix the vulnerability."
- },
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
- "identifiers": {
- "type": "array",
- "minItems": 1,
- "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
- "items": {
- "type": "object",
- "required": [
- "type",
- "name",
- "value"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
- "minLength": 1
- },
- "name": {
- "type": "string",
- "description": "Human-readable name of the identifier.",
- "minLength": 1
- },
- "url": {
- "type": "string",
- "description": "URL of the identifier's documentation.",
- "format": "uri"
- },
- "value": {
- "type": "string",
- "description": "Value of the identifier, for matching purpose.",
- "minLength": 1
- }
- }
- }
- },
- "links": {
- "type": "array",
- "description": "An array of references to external documentation or articles that describe the vulnerability.",
- "items": {
- "type": "object",
- "required": [
- "url"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the vulnerability details link."
- },
- "url": {
- "type": "string",
- "description": "URL of the vulnerability details document.",
- "format": "uri"
- }
- }
- }
- },
- "details": {
- "$ref": "#/definitions/named_list/properties/items"
- },
- "tracking": {
- "description": "Describes how this vulnerability should be tracked as the project changes.",
- "oneOf": [
- {
- "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
- "required": [
- "items"
- ],
- "properties": {
- "type": {
- "const": "source"
- },
- "items": {
- "type": "array",
- "items": {
- "description": "An item that should be tracked using source-specific tracking methods.",
- "type": "object",
- "required": [
- "signatures"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the file that includes the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the file that includes the vulnerability."
- },
- "signatures": {
- "type": "array",
- "description": "An array of calculated tracking signatures for this tracking item.",
- "minItems": 1,
- "items": {
- "description": "A calculated tracking signature value and metadata.",
- "required": [
- "algorithm",
- "value"
- ],
- "properties": {
- "algorithm": {
- "type": "string",
- "description": "The algorithm used to generate the signature."
- },
- "value": {
- "type": "string",
- "description": "The result of this signature algorithm."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "Each tracking type must declare its own type."
- }
- }
- },
- "flags": {
- "description": "Flags that can be attached to vulnerabilities.",
- "type": "array",
- "items": {
- "type": "object",
- "description": "Informational flags identified and assigned to a vulnerability.",
- "required": [
- "type",
- "origin",
- "description"
- ],
- "properties": {
- "type": {
- "type": "string",
- "minLength": 1,
- "description": "Result of the scan.",
- "enum": [
- "flagged-as-likely-false-positive"
- ]
- },
- "origin": {
- "minLength": 1,
- "description": "Tool that issued the flag.",
- "type": "string"
- },
- "description": {
- "minLength": 1,
- "description": "What the flag is about.",
- "type": "string"
- }
- }
- }
- },
- "location": {
- "required": [
- "commit"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located"
- },
- "commit": {
- "type": "object",
- "description": "Represents the commit in which the vulnerability was detected",
- "required": [
- "sha"
- ],
- "properties": {
- "author": {
- "type": "string"
- },
- "date": {
- "type": "string"
- },
- "message": {
- "type": "string"
- },
- "sha": {
- "type": "string",
- "minLength": 1
- }
- }
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the code affected by the vulnerability"
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the code affected by the vulnerability"
- },
- "class": {
- "type": "string",
- "description": "Provides the name of the class where the vulnerability is located"
- },
- "method": {
- "type": "string",
- "description": "Provides the name of the method where the vulnerability is located"
- }
- }
- },
- "raw_source_code_extract": {
- "type": "string",
- "description": "Provides an unsanitized excerpt of the affected source code."
- }
- }
- }
- },
- "remediations": {
- "type": "array",
- "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
- "items": {
- "type": "object",
- "required": [
- "fixes",
- "summary",
- "diff"
- ],
- "properties": {
- "fixes": {
- "type": "array",
- "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
- "items": {
- "type": "object",
- "required": [
- "cve"
- ],
- "properties": {
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- }
- }
- }
- },
- "summary": {
- "type": "string",
- "minLength": 1,
- "description": "An overview of how the vulnerabilities were fixed."
- },
- "diff": {
- "type": "string",
- "minLength": 1,
- "description": "A base64-encoded remediation code diff, compatible with git apply."
- }
- }
- }
- }
- }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/cluster-image-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/cluster-image-scanning-report-format.json
deleted file mode 100644
index 31840a7e914..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/cluster-image-scanning-report-format.json
+++ /dev/null
@@ -1,977 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Report format for GitLab Cluster Image Scanning",
- "description": "This schema provides the the report format for Cluster Image Scanning (https://docs.gitlab.com/ee/user/application_security/cluster_image_scanning/).",
- "definitions": {
- "detail_type": {
- "oneOf": [
- {
- "$ref": "#/definitions/named_list"
- },
- {
- "$ref": "#/definitions/list"
- },
- {
- "$ref": "#/definitions/table"
- },
- {
- "$ref": "#/definitions/text"
- },
- {
- "$ref": "#/definitions/url"
- },
- {
- "$ref": "#/definitions/code"
- },
- {
- "$ref": "#/definitions/value"
- },
- {
- "$ref": "#/definitions/diff"
- },
- {
- "$ref": "#/definitions/markdown"
- },
- {
- "$ref": "#/definitions/commit"
- },
- {
- "$ref": "#/definitions/file_location"
- },
- {
- "$ref": "#/definitions/module_location"
- }
- ]
- },
- "text_value": {
- "type": "string"
- },
- "named_field": {
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "$ref": "#/definitions/text_value",
- "minLength": 1
- },
- "description": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "named_list": {
- "type": "object",
- "description": "An object with named and typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "named-list"
- },
- "items": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- {
- "$ref": "#/definitions/named_field"
- },
- {
- "$ref": "#/definitions/detail_type"
- }
- ]
- }
- }
- }
- }
- },
- "list": {
- "type": "object",
- "description": "A list of typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- },
- "table": {
- "type": "object",
- "description": "A table of typed fields",
- "required": [
- "type",
- "rows"
- ],
- "properties": {
- "type": {
- "const": "table"
- },
- "header": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- },
- "rows": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- }
- },
- "text": {
- "type": "object",
- "description": "Raw text",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "text"
- },
- "value": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "url": {
- "type": "object",
- "description": "A single URL",
- "required": [
- "type",
- "href"
- ],
- "properties": {
- "type": {
- "const": "url"
- },
- "text": {
- "$ref": "#/definitions/text_value"
- },
- "href": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "http://mysite.com"
- ]
- }
- }
- },
- "code": {
- "type": "object",
- "description": "A codeblock",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "code"
- },
- "value": {
- "type": "string"
- },
- "lang": {
- "type": "string",
- "description": "A programming language"
- }
- }
- },
- "value": {
- "type": "object",
- "description": "A field that can store a range of types of value",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "value"
- },
- "value": {
- "type": [
- "number",
- "string",
- "boolean"
- ]
- }
- }
- },
- "diff": {
- "type": "object",
- "description": "A diff",
- "required": [
- "type",
- "before",
- "after"
- ],
- "properties": {
- "type": {
- "const": "diff"
- },
- "before": {
- "type": "string"
- },
- "after": {
- "type": "string"
- }
- }
- },
- "markdown": {
- "type": "object",
- "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "markdown"
- },
- "value": {
- "$ref": "#/definitions/text_value",
- "examples": [
- "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
- ]
- }
- }
- },
- "commit": {
- "type": "object",
- "description": "A commit/tag/branch within the GitLab project",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "commit"
- },
- "value": {
- "type": "string",
- "description": "The commit SHA",
- "minLength": 1
- }
- }
- },
- "file_location": {
- "type": "object",
- "description": "A location within a file in the project",
- "required": [
- "type",
- "file_name",
- "line_start"
- ],
- "properties": {
- "type": {
- "const": "file-location"
- },
- "file_name": {
- "type": "string",
- "minLength": 1
- },
- "line_start": {
- "type": "integer"
- },
- "line_end": {
- "type": "integer"
- }
- }
- },
- "module_location": {
- "type": "object",
- "description": "A location within a binary module of the form module+relative_offset",
- "required": [
- "type",
- "module_name",
- "offset"
- ],
- "properties": {
- "type": {
- "const": "module-location"
- },
- "module_name": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "compiled_binary"
- ]
- },
- "offset": {
- "type": "integer",
- "examples": [
- 100
- ]
- }
- }
- }
- },
- "self": {
- "version": "14.1.2"
- },
- "required": [
- "version",
- "vulnerabilities"
- ],
- "additionalProperties": true,
- "properties": {
- "scan": {
- "type": "object",
- "required": [
- "end_time",
- "scanner",
- "start_time",
- "status",
- "type"
- ],
- "properties": {
- "end_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-01-28T03:26:02"
- ]
- },
- "messages": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Communication intended for the initiator of a scan.",
- "required": [
- "level",
- "value"
- ],
- "properties": {
- "level": {
- "type": "string",
- "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
- "enum": [
- "info",
- "warn",
- "fatal"
- ],
- "examples": [
- "info"
- ]
- },
- "value": {
- "type": "string",
- "description": "The message to communicate.",
- "minLength": 1,
- "examples": [
- "Permission denied, scanning aborted"
- ]
- }
- }
- }
- },
- "analyzer": {
- "type": "object",
- "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the analyzer.",
- "minLength": 1,
- "examples": [
- "gitlab-dast"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the analyzer, not required to be unique.",
- "minLength": 1,
- "examples": [
- "GitLab DAST"
- ]
- },
- "url": {
- "type": "string",
- "format": "uri",
- "pattern": "^https?://.+",
- "description": "A link to more information about the analyzer.",
- "examples": [
- "https://docs.gitlab.com/ee/user/application_security/dast"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the analyzer.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- },
- "version": {
- "type": "string",
- "description": "The version of the analyzer.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- }
- }
- },
- "scanner": {
- "type": "object",
- "description": "Object defining the scanner used to perform the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the scanner.",
- "minLength": 1,
- "examples": [
- "my-sast-scanner"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the scanner, not required to be unique.",
- "minLength": 1,
- "examples": [
- "My SAST Scanner"
- ]
- },
- "url": {
- "type": "string",
- "description": "A link to more information about the scanner.",
- "examples": [
- "https://scanner.url"
- ]
- },
- "version": {
- "type": "string",
- "description": "The version of the scanner.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the scanner.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- }
- }
- },
- "start_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-02-14T16:01:59"
- ]
- },
- "status": {
- "type": "string",
- "description": "Result of the scan.",
- "enum": [
- "success",
- "failure"
- ]
- },
- "type": {
- "type": "string",
- "description": "Type of the scan.",
- "enum": [
- "cluster_image_scanning"
- ]
- }
- }
- },
- "schema": {
- "type": "string",
- "description": "URI pointing to the validating security report schema.",
- "format": "uri"
- },
- "version": {
- "type": "string",
- "description": "The version of the schema to which the JSON report conforms.",
- "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
- },
- "vulnerabilities": {
- "type": "array",
- "description": "Array of vulnerability objects.",
- "items": {
- "type": "object",
- "description": "Describes the vulnerability using GitLab Flavored Markdown",
- "required": [
- "category",
- "cve",
- "identifiers",
- "location",
- "scanner"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
- "examples": [
- "642735a5-1425-428d-8d4e-3c854885a3c9"
- ]
- },
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
- "name": {
- "type": "string",
- "description": "The name of the vulnerability. This must not include the finding's specific information."
- },
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
- "description": {
- "type": "string",
- "description": "A long text section describing the vulnerability more fully."
- },
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
- "severity": {
- "type": "string",
- "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Info",
- "Unknown",
- "Low",
- "Medium",
- "High",
- "Critical"
- ]
- },
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
- "solution": {
- "type": "string",
- "description": "Explanation of how to fix the vulnerability."
- },
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
- "identifiers": {
- "type": "array",
- "minItems": 1,
- "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
- "items": {
- "type": "object",
- "required": [
- "type",
- "name",
- "value"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
- "minLength": 1
- },
- "name": {
- "type": "string",
- "description": "Human-readable name of the identifier.",
- "minLength": 1
- },
- "url": {
- "type": "string",
- "description": "URL of the identifier's documentation.",
- "format": "uri"
- },
- "value": {
- "type": "string",
- "description": "Value of the identifier, for matching purpose.",
- "minLength": 1
- }
- }
- }
- },
- "links": {
- "type": "array",
- "description": "An array of references to external documentation or articles that describe the vulnerability.",
- "items": {
- "type": "object",
- "required": [
- "url"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the vulnerability details link."
- },
- "url": {
- "type": "string",
- "description": "URL of the vulnerability details document.",
- "format": "uri"
- }
- }
- }
- },
- "details": {
- "$ref": "#/definitions/named_list/properties/items"
- },
- "tracking": {
- "description": "Describes how this vulnerability should be tracked as the project changes.",
- "oneOf": [
- {
- "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
- "required": [
- "items"
- ],
- "properties": {
- "type": {
- "const": "source"
- },
- "items": {
- "type": "array",
- "items": {
- "description": "An item that should be tracked using source-specific tracking methods.",
- "type": "object",
- "required": [
- "signatures"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the file that includes the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the file that includes the vulnerability."
- },
- "signatures": {
- "type": "array",
- "description": "An array of calculated tracking signatures for this tracking item.",
- "minItems": 1,
- "items": {
- "description": "A calculated tracking signature value and metadata.",
- "required": [
- "algorithm",
- "value"
- ],
- "properties": {
- "algorithm": {
- "type": "string",
- "description": "The algorithm used to generate the signature."
- },
- "value": {
- "type": "string",
- "description": "The result of this signature algorithm."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "Each tracking type must declare its own type."
- }
- }
- },
- "flags": {
- "description": "Flags that can be attached to vulnerabilities.",
- "type": "array",
- "items": {
- "type": "object",
- "description": "Informational flags identified and assigned to a vulnerability.",
- "required": [
- "type",
- "origin",
- "description"
- ],
- "properties": {
- "type": {
- "type": "string",
- "minLength": 1,
- "description": "Result of the scan.",
- "enum": [
- "flagged-as-likely-false-positive"
- ]
- },
- "origin": {
- "minLength": 1,
- "description": "Tool that issued the flag.",
- "type": "string"
- },
- "description": {
- "minLength": 1,
- "description": "What the flag is about.",
- "type": "string"
- }
- }
- }
- },
- "location": {
- "type": "object",
- "description": "Identifies the vulnerability's location.",
- "required": [
- "dependency",
- "image",
- "kubernetes_resource"
- ],
- "properties": {
- "dependency": {
- "type": "object",
- "description": "Describes the dependency of a project where the vulnerability is located.",
- "properties": {
- "package": {
- "type": "object",
- "description": "Provides information on the package where the vulnerability is located.",
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the package where the vulnerability is located."
- }
- }
- },
- "version": {
- "type": "string",
- "description": "Version of the vulnerable package."
- },
- "iid": {
- "description": "ID that identifies the dependency in the scope of a dependency file.",
- "type": "number"
- },
- "direct": {
- "type": "boolean",
- "description": "Tells whether this is a direct, top-level dependency of the scanned project."
- },
- "dependency_path": {
- "type": "array",
- "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.",
- "items": {
- "type": "object",
- "required": [
- "iid"
- ],
- "properties": {
- "iid": {
- "type": "number",
- "description": "ID that is unique in the scope of a parent object, and specific to the resource type."
- }
- }
- }
- }
- }
- },
- "operating_system": {
- "type": "string",
- "minLength": 1,
- "maxLength": 255,
- "description": "The operating system that contains the vulnerable package."
- },
- "image": {
- "type": "string",
- "minLength": 1,
- "description": "The analyzed Docker image.",
- "examples": [
- "index.docker.io/library/nginx:1.21"
- ]
- },
- "kubernetes_resource": {
- "type": "object",
- "description": "The specific Kubernetes resource that was scanned.",
- "required": [
- "namespace",
- "kind",
- "name",
- "container_name"
- ],
- "properties": {
- "namespace": {
- "type": "string",
- "minLength": 1,
- "maxLength": 255,
- "description": "The Kubernetes namespace the resource that had its image scanned.",
- "examples": [
- "default",
- "staging",
- "production"
- ]
- },
- "kind": {
- "type": "string",
- "minLength": 1,
- "maxLength": 255,
- "description": "The Kubernetes kind the resource that had its image scanned.",
- "examples": [
- "Deployment",
- "DaemonSet"
- ]
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "maxLength": 255,
- "description": "The name of the resource that had its image scanned.",
- "examples": [
- "nginx-ingress"
- ]
- },
- "container_name": {
- "type": "string",
- "minLength": 1,
- "maxLength": 255,
- "description": "The name of the container that had its image scanned.",
- "examples": [
- "nginx"
- ]
- },
- "agent_id": {
- "type": "string",
- "minLength": 1,
- "maxLength": 255,
- "description": "The GitLab ID of the Kubernetes Agent which performed the scan.",
- "examples": [
- "1234"
- ]
- },
- "cluster_id": {
- "type": "string",
- "minLength": 1,
- "maxLength": 255,
- "description": "The GitLab ID of the Kubernetes cluster when using cluster integration.",
- "examples": [
- "1234"
- ]
- }
- }
- }
- }
- }
- }
- }
- },
- "remediations": {
- "type": "array",
- "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
- "items": {
- "type": "object",
- "required": [
- "fixes",
- "summary",
- "diff"
- ],
- "properties": {
- "fixes": {
- "type": "array",
- "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
- "items": {
- "type": "object",
- "required": [
- "cve"
- ],
- "properties": {
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- }
- }
- }
- },
- "summary": {
- "type": "string",
- "minLength": 1,
- "description": "An overview of how the vulnerabilities were fixed."
- },
- "diff": {
- "type": "string",
- "minLength": 1,
- "description": "A base64-encoded remediation code diff, compatible with git apply."
- }
- }
- }
- }
- }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/container-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/container-scanning-report-format.json
deleted file mode 100644
index c70628a0949..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/container-scanning-report-format.json
+++ /dev/null
@@ -1,911 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Report format for GitLab Container Scanning",
- "description": "This schema provides the the report format for Container Scanning (https://docs.gitlab.com/ee/user/application_security/container_scanning).",
- "definitions": {
- "detail_type": {
- "oneOf": [
- {
- "$ref": "#/definitions/named_list"
- },
- {
- "$ref": "#/definitions/list"
- },
- {
- "$ref": "#/definitions/table"
- },
- {
- "$ref": "#/definitions/text"
- },
- {
- "$ref": "#/definitions/url"
- },
- {
- "$ref": "#/definitions/code"
- },
- {
- "$ref": "#/definitions/value"
- },
- {
- "$ref": "#/definitions/diff"
- },
- {
- "$ref": "#/definitions/markdown"
- },
- {
- "$ref": "#/definitions/commit"
- },
- {
- "$ref": "#/definitions/file_location"
- },
- {
- "$ref": "#/definitions/module_location"
- }
- ]
- },
- "text_value": {
- "type": "string"
- },
- "named_field": {
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "$ref": "#/definitions/text_value",
- "minLength": 1
- },
- "description": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "named_list": {
- "type": "object",
- "description": "An object with named and typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "named-list"
- },
- "items": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- {
- "$ref": "#/definitions/named_field"
- },
- {
- "$ref": "#/definitions/detail_type"
- }
- ]
- }
- }
- }
- }
- },
- "list": {
- "type": "object",
- "description": "A list of typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- },
- "table": {
- "type": "object",
- "description": "A table of typed fields",
- "required": [
- "type",
- "rows"
- ],
- "properties": {
- "type": {
- "const": "table"
- },
- "header": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- },
- "rows": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- }
- },
- "text": {
- "type": "object",
- "description": "Raw text",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "text"
- },
- "value": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "url": {
- "type": "object",
- "description": "A single URL",
- "required": [
- "type",
- "href"
- ],
- "properties": {
- "type": {
- "const": "url"
- },
- "text": {
- "$ref": "#/definitions/text_value"
- },
- "href": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "http://mysite.com"
- ]
- }
- }
- },
- "code": {
- "type": "object",
- "description": "A codeblock",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "code"
- },
- "value": {
- "type": "string"
- },
- "lang": {
- "type": "string",
- "description": "A programming language"
- }
- }
- },
- "value": {
- "type": "object",
- "description": "A field that can store a range of types of value",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "value"
- },
- "value": {
- "type": [
- "number",
- "string",
- "boolean"
- ]
- }
- }
- },
- "diff": {
- "type": "object",
- "description": "A diff",
- "required": [
- "type",
- "before",
- "after"
- ],
- "properties": {
- "type": {
- "const": "diff"
- },
- "before": {
- "type": "string"
- },
- "after": {
- "type": "string"
- }
- }
- },
- "markdown": {
- "type": "object",
- "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "markdown"
- },
- "value": {
- "$ref": "#/definitions/text_value",
- "examples": [
- "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
- ]
- }
- }
- },
- "commit": {
- "type": "object",
- "description": "A commit/tag/branch within the GitLab project",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "commit"
- },
- "value": {
- "type": "string",
- "description": "The commit SHA",
- "minLength": 1
- }
- }
- },
- "file_location": {
- "type": "object",
- "description": "A location within a file in the project",
- "required": [
- "type",
- "file_name",
- "line_start"
- ],
- "properties": {
- "type": {
- "const": "file-location"
- },
- "file_name": {
- "type": "string",
- "minLength": 1
- },
- "line_start": {
- "type": "integer"
- },
- "line_end": {
- "type": "integer"
- }
- }
- },
- "module_location": {
- "type": "object",
- "description": "A location within a binary module of the form module+relative_offset",
- "required": [
- "type",
- "module_name",
- "offset"
- ],
- "properties": {
- "type": {
- "const": "module-location"
- },
- "module_name": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "compiled_binary"
- ]
- },
- "offset": {
- "type": "integer",
- "examples": [
- 100
- ]
- }
- }
- }
- },
- "self": {
- "version": "14.1.2"
- },
- "required": [
- "version",
- "vulnerabilities"
- ],
- "additionalProperties": true,
- "properties": {
- "scan": {
- "type": "object",
- "required": [
- "end_time",
- "scanner",
- "start_time",
- "status",
- "type"
- ],
- "properties": {
- "end_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-01-28T03:26:02"
- ]
- },
- "messages": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Communication intended for the initiator of a scan.",
- "required": [
- "level",
- "value"
- ],
- "properties": {
- "level": {
- "type": "string",
- "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
- "enum": [
- "info",
- "warn",
- "fatal"
- ],
- "examples": [
- "info"
- ]
- },
- "value": {
- "type": "string",
- "description": "The message to communicate.",
- "minLength": 1,
- "examples": [
- "Permission denied, scanning aborted"
- ]
- }
- }
- }
- },
- "analyzer": {
- "type": "object",
- "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the analyzer.",
- "minLength": 1,
- "examples": [
- "gitlab-dast"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the analyzer, not required to be unique.",
- "minLength": 1,
- "examples": [
- "GitLab DAST"
- ]
- },
- "url": {
- "type": "string",
- "format": "uri",
- "pattern": "^https?://.+",
- "description": "A link to more information about the analyzer.",
- "examples": [
- "https://docs.gitlab.com/ee/user/application_security/dast"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the analyzer.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- },
- "version": {
- "type": "string",
- "description": "The version of the analyzer.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- }
- }
- },
- "scanner": {
- "type": "object",
- "description": "Object defining the scanner used to perform the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the scanner.",
- "minLength": 1,
- "examples": [
- "my-sast-scanner"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the scanner, not required to be unique.",
- "minLength": 1,
- "examples": [
- "My SAST Scanner"
- ]
- },
- "url": {
- "type": "string",
- "description": "A link to more information about the scanner.",
- "examples": [
- "https://scanner.url"
- ]
- },
- "version": {
- "type": "string",
- "description": "The version of the scanner.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the scanner.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- }
- }
- },
- "start_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-02-14T16:01:59"
- ]
- },
- "status": {
- "type": "string",
- "description": "Result of the scan.",
- "enum": [
- "success",
- "failure"
- ]
- },
- "type": {
- "type": "string",
- "description": "Type of the scan.",
- "enum": [
- "container_scanning"
- ]
- }
- }
- },
- "schema": {
- "type": "string",
- "description": "URI pointing to the validating security report schema.",
- "format": "uri"
- },
- "version": {
- "type": "string",
- "description": "The version of the schema to which the JSON report conforms.",
- "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
- },
- "vulnerabilities": {
- "type": "array",
- "description": "Array of vulnerability objects.",
- "items": {
- "type": "object",
- "description": "Describes the vulnerability using GitLab Flavored Markdown",
- "required": [
- "category",
- "cve",
- "identifiers",
- "location",
- "scanner"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
- "examples": [
- "642735a5-1425-428d-8d4e-3c854885a3c9"
- ]
- },
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
- "name": {
- "type": "string",
- "description": "The name of the vulnerability. This must not include the finding's specific information."
- },
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
- "description": {
- "type": "string",
- "description": "A long text section describing the vulnerability more fully."
- },
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
- "severity": {
- "type": "string",
- "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Info",
- "Unknown",
- "Low",
- "Medium",
- "High",
- "Critical"
- ]
- },
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
- "solution": {
- "type": "string",
- "description": "Explanation of how to fix the vulnerability."
- },
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
- "identifiers": {
- "type": "array",
- "minItems": 1,
- "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
- "items": {
- "type": "object",
- "required": [
- "type",
- "name",
- "value"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
- "minLength": 1
- },
- "name": {
- "type": "string",
- "description": "Human-readable name of the identifier.",
- "minLength": 1
- },
- "url": {
- "type": "string",
- "description": "URL of the identifier's documentation.",
- "format": "uri"
- },
- "value": {
- "type": "string",
- "description": "Value of the identifier, for matching purpose.",
- "minLength": 1
- }
- }
- }
- },
- "links": {
- "type": "array",
- "description": "An array of references to external documentation or articles that describe the vulnerability.",
- "items": {
- "type": "object",
- "required": [
- "url"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the vulnerability details link."
- },
- "url": {
- "type": "string",
- "description": "URL of the vulnerability details document.",
- "format": "uri"
- }
- }
- }
- },
- "details": {
- "$ref": "#/definitions/named_list/properties/items"
- },
- "tracking": {
- "description": "Describes how this vulnerability should be tracked as the project changes.",
- "oneOf": [
- {
- "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
- "required": [
- "items"
- ],
- "properties": {
- "type": {
- "const": "source"
- },
- "items": {
- "type": "array",
- "items": {
- "description": "An item that should be tracked using source-specific tracking methods.",
- "type": "object",
- "required": [
- "signatures"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the file that includes the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the file that includes the vulnerability."
- },
- "signatures": {
- "type": "array",
- "description": "An array of calculated tracking signatures for this tracking item.",
- "minItems": 1,
- "items": {
- "description": "A calculated tracking signature value and metadata.",
- "required": [
- "algorithm",
- "value"
- ],
- "properties": {
- "algorithm": {
- "type": "string",
- "description": "The algorithm used to generate the signature."
- },
- "value": {
- "type": "string",
- "description": "The result of this signature algorithm."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "Each tracking type must declare its own type."
- }
- }
- },
- "flags": {
- "description": "Flags that can be attached to vulnerabilities.",
- "type": "array",
- "items": {
- "type": "object",
- "description": "Informational flags identified and assigned to a vulnerability.",
- "required": [
- "type",
- "origin",
- "description"
- ],
- "properties": {
- "type": {
- "type": "string",
- "minLength": 1,
- "description": "Result of the scan.",
- "enum": [
- "flagged-as-likely-false-positive"
- ]
- },
- "origin": {
- "minLength": 1,
- "description": "Tool that issued the flag.",
- "type": "string"
- },
- "description": {
- "minLength": 1,
- "description": "What the flag is about.",
- "type": "string"
- }
- }
- }
- },
- "location": {
- "type": "object",
- "description": "Identifies the vulnerability's location.",
- "required": [
- "dependency",
- "operating_system",
- "image"
- ],
- "properties": {
- "dependency": {
- "type": "object",
- "description": "Describes the dependency of a project where the vulnerability is located.",
- "properties": {
- "package": {
- "type": "object",
- "description": "Provides information on the package where the vulnerability is located.",
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the package where the vulnerability is located."
- }
- }
- },
- "version": {
- "type": "string",
- "description": "Version of the vulnerable package."
- },
- "iid": {
- "description": "ID that identifies the dependency in the scope of a dependency file.",
- "type": "number"
- },
- "direct": {
- "type": "boolean",
- "description": "Tells whether this is a direct, top-level dependency of the scanned project."
- },
- "dependency_path": {
- "type": "array",
- "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.",
- "items": {
- "type": "object",
- "required": [
- "iid"
- ],
- "properties": {
- "iid": {
- "type": "number",
- "description": "ID that is unique in the scope of a parent object, and specific to the resource type."
- }
- }
- }
- }
- }
- },
- "operating_system": {
- "type": "string",
- "minLength": 1,
- "description": "The operating system that contains the vulnerable package."
- },
- "image": {
- "type": "string",
- "minLength": 1,
- "pattern": "^[^:]+(:\\d+[^:]*)?:[^:]+$",
- "description": "The analyzed Docker image."
- },
- "default_branch_image": {
- "type": "string",
- "maxLength": 255,
- "pattern": "^[a-zA-Z0-9/_.-]+(:\\d+[a-zA-Z0-9/_.-]*)?:[a-zA-Z0-9_.-]+$",
- "description": "The name of the image on the default branch."
- }
- }
- }
- }
- }
- },
- "remediations": {
- "type": "array",
- "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
- "items": {
- "type": "object",
- "required": [
- "fixes",
- "summary",
- "diff"
- ],
- "properties": {
- "fixes": {
- "type": "array",
- "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
- "items": {
- "type": "object",
- "required": [
- "cve"
- ],
- "properties": {
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- }
- }
- }
- },
- "summary": {
- "type": "string",
- "minLength": 1,
- "description": "An overview of how the vulnerabilities were fixed."
- },
- "diff": {
- "type": "string",
- "minLength": 1,
- "description": "A base64-encoded remediation code diff, compatible with git apply."
- }
- }
- }
- }
- }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/coverage-fuzzing-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/coverage-fuzzing-report-format.json
deleted file mode 100644
index fbc7b4ea733..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/coverage-fuzzing-report-format.json
+++ /dev/null
@@ -1,874 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Report format for GitLab Fuzz Testing",
- "description": "This schema provides the report format for Coverage Guided Fuzz Testing (https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing).",
- "definitions": {
- "detail_type": {
- "oneOf": [
- {
- "$ref": "#/definitions/named_list"
- },
- {
- "$ref": "#/definitions/list"
- },
- {
- "$ref": "#/definitions/table"
- },
- {
- "$ref": "#/definitions/text"
- },
- {
- "$ref": "#/definitions/url"
- },
- {
- "$ref": "#/definitions/code"
- },
- {
- "$ref": "#/definitions/value"
- },
- {
- "$ref": "#/definitions/diff"
- },
- {
- "$ref": "#/definitions/markdown"
- },
- {
- "$ref": "#/definitions/commit"
- },
- {
- "$ref": "#/definitions/file_location"
- },
- {
- "$ref": "#/definitions/module_location"
- }
- ]
- },
- "text_value": {
- "type": "string"
- },
- "named_field": {
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "$ref": "#/definitions/text_value",
- "minLength": 1
- },
- "description": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "named_list": {
- "type": "object",
- "description": "An object with named and typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "named-list"
- },
- "items": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- {
- "$ref": "#/definitions/named_field"
- },
- {
- "$ref": "#/definitions/detail_type"
- }
- ]
- }
- }
- }
- }
- },
- "list": {
- "type": "object",
- "description": "A list of typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- },
- "table": {
- "type": "object",
- "description": "A table of typed fields",
- "required": [
- "type",
- "rows"
- ],
- "properties": {
- "type": {
- "const": "table"
- },
- "header": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- },
- "rows": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- }
- },
- "text": {
- "type": "object",
- "description": "Raw text",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "text"
- },
- "value": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "url": {
- "type": "object",
- "description": "A single URL",
- "required": [
- "type",
- "href"
- ],
- "properties": {
- "type": {
- "const": "url"
- },
- "text": {
- "$ref": "#/definitions/text_value"
- },
- "href": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "http://mysite.com"
- ]
- }
- }
- },
- "code": {
- "type": "object",
- "description": "A codeblock",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "code"
- },
- "value": {
- "type": "string"
- },
- "lang": {
- "type": "string",
- "description": "A programming language"
- }
- }
- },
- "value": {
- "type": "object",
- "description": "A field that can store a range of types of value",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "value"
- },
- "value": {
- "type": [
- "number",
- "string",
- "boolean"
- ]
- }
- }
- },
- "diff": {
- "type": "object",
- "description": "A diff",
- "required": [
- "type",
- "before",
- "after"
- ],
- "properties": {
- "type": {
- "const": "diff"
- },
- "before": {
- "type": "string"
- },
- "after": {
- "type": "string"
- }
- }
- },
- "markdown": {
- "type": "object",
- "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "markdown"
- },
- "value": {
- "$ref": "#/definitions/text_value",
- "examples": [
- "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
- ]
- }
- }
- },
- "commit": {
- "type": "object",
- "description": "A commit/tag/branch within the GitLab project",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "commit"
- },
- "value": {
- "type": "string",
- "description": "The commit SHA",
- "minLength": 1
- }
- }
- },
- "file_location": {
- "type": "object",
- "description": "A location within a file in the project",
- "required": [
- "type",
- "file_name",
- "line_start"
- ],
- "properties": {
- "type": {
- "const": "file-location"
- },
- "file_name": {
- "type": "string",
- "minLength": 1
- },
- "line_start": {
- "type": "integer"
- },
- "line_end": {
- "type": "integer"
- }
- }
- },
- "module_location": {
- "type": "object",
- "description": "A location within a binary module of the form module+relative_offset",
- "required": [
- "type",
- "module_name",
- "offset"
- ],
- "properties": {
- "type": {
- "const": "module-location"
- },
- "module_name": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "compiled_binary"
- ]
- },
- "offset": {
- "type": "integer",
- "examples": [
- 100
- ]
- }
- }
- }
- },
- "self": {
- "version": "14.1.2"
- },
- "required": [
- "version",
- "vulnerabilities"
- ],
- "additionalProperties": true,
- "properties": {
- "scan": {
- "type": "object",
- "required": [
- "end_time",
- "scanner",
- "start_time",
- "status",
- "type"
- ],
- "properties": {
- "end_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-01-28T03:26:02"
- ]
- },
- "messages": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Communication intended for the initiator of a scan.",
- "required": [
- "level",
- "value"
- ],
- "properties": {
- "level": {
- "type": "string",
- "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
- "enum": [
- "info",
- "warn",
- "fatal"
- ],
- "examples": [
- "info"
- ]
- },
- "value": {
- "type": "string",
- "description": "The message to communicate.",
- "minLength": 1,
- "examples": [
- "Permission denied, scanning aborted"
- ]
- }
- }
- }
- },
- "analyzer": {
- "type": "object",
- "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the analyzer.",
- "minLength": 1,
- "examples": [
- "gitlab-dast"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the analyzer, not required to be unique.",
- "minLength": 1,
- "examples": [
- "GitLab DAST"
- ]
- },
- "url": {
- "type": "string",
- "format": "uri",
- "pattern": "^https?://.+",
- "description": "A link to more information about the analyzer.",
- "examples": [
- "https://docs.gitlab.com/ee/user/application_security/dast"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the analyzer.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- },
- "version": {
- "type": "string",
- "description": "The version of the analyzer.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- }
- }
- },
- "scanner": {
- "type": "object",
- "description": "Object defining the scanner used to perform the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the scanner.",
- "minLength": 1,
- "examples": [
- "my-sast-scanner"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the scanner, not required to be unique.",
- "minLength": 1,
- "examples": [
- "My SAST Scanner"
- ]
- },
- "url": {
- "type": "string",
- "description": "A link to more information about the scanner.",
- "examples": [
- "https://scanner.url"
- ]
- },
- "version": {
- "type": "string",
- "description": "The version of the scanner.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the scanner.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- }
- }
- },
- "start_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-02-14T16:01:59"
- ]
- },
- "status": {
- "type": "string",
- "description": "Result of the scan.",
- "enum": [
- "success",
- "failure"
- ]
- },
- "type": {
- "type": "string",
- "description": "Type of the scan.",
- "enum": [
- "coverage_fuzzing"
- ]
- }
- }
- },
- "schema": {
- "type": "string",
- "description": "URI pointing to the validating security report schema.",
- "format": "uri"
- },
- "version": {
- "type": "string",
- "description": "The version of the schema to which the JSON report conforms.",
- "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
- },
- "vulnerabilities": {
- "type": "array",
- "description": "Array of vulnerability objects.",
- "items": {
- "type": "object",
- "description": "Describes the vulnerability using GitLab Flavored Markdown",
- "required": [
- "category",
- "cve",
- "identifiers",
- "location",
- "scanner"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
- "examples": [
- "642735a5-1425-428d-8d4e-3c854885a3c9"
- ]
- },
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
- "name": {
- "type": "string",
- "description": "The name of the vulnerability. This must not include the finding's specific information."
- },
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
- "description": {
- "type": "string",
- "description": "A long text section describing the vulnerability more fully."
- },
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
- "severity": {
- "type": "string",
- "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Info",
- "Unknown",
- "Low",
- "Medium",
- "High",
- "Critical"
- ]
- },
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
- "solution": {
- "type": "string",
- "description": "Explanation of how to fix the vulnerability."
- },
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
- "identifiers": {
- "type": "array",
- "minItems": 1,
- "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
- "items": {
- "type": "object",
- "required": [
- "type",
- "name",
- "value"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
- "minLength": 1
- },
- "name": {
- "type": "string",
- "description": "Human-readable name of the identifier.",
- "minLength": 1
- },
- "url": {
- "type": "string",
- "description": "URL of the identifier's documentation.",
- "format": "uri"
- },
- "value": {
- "type": "string",
- "description": "Value of the identifier, for matching purpose.",
- "minLength": 1
- }
- }
- }
- },
- "links": {
- "type": "array",
- "description": "An array of references to external documentation or articles that describe the vulnerability.",
- "items": {
- "type": "object",
- "required": [
- "url"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the vulnerability details link."
- },
- "url": {
- "type": "string",
- "description": "URL of the vulnerability details document.",
- "format": "uri"
- }
- }
- }
- },
- "details": {
- "$ref": "#/definitions/named_list/properties/items"
- },
- "tracking": {
- "description": "Describes how this vulnerability should be tracked as the project changes.",
- "oneOf": [
- {
- "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
- "required": [
- "items"
- ],
- "properties": {
- "type": {
- "const": "source"
- },
- "items": {
- "type": "array",
- "items": {
- "description": "An item that should be tracked using source-specific tracking methods.",
- "type": "object",
- "required": [
- "signatures"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the file that includes the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the file that includes the vulnerability."
- },
- "signatures": {
- "type": "array",
- "description": "An array of calculated tracking signatures for this tracking item.",
- "minItems": 1,
- "items": {
- "description": "A calculated tracking signature value and metadata.",
- "required": [
- "algorithm",
- "value"
- ],
- "properties": {
- "algorithm": {
- "type": "string",
- "description": "The algorithm used to generate the signature."
- },
- "value": {
- "type": "string",
- "description": "The result of this signature algorithm."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "Each tracking type must declare its own type."
- }
- }
- },
- "flags": {
- "description": "Flags that can be attached to vulnerabilities.",
- "type": "array",
- "items": {
- "type": "object",
- "description": "Informational flags identified and assigned to a vulnerability.",
- "required": [
- "type",
- "origin",
- "description"
- ],
- "properties": {
- "type": {
- "type": "string",
- "minLength": 1,
- "description": "Result of the scan.",
- "enum": [
- "flagged-as-likely-false-positive"
- ]
- },
- "origin": {
- "minLength": 1,
- "description": "Tool that issued the flag.",
- "type": "string"
- },
- "description": {
- "minLength": 1,
- "description": "What the flag is about.",
- "type": "string"
- }
- }
- }
- },
- "location": {
- "description": "The location of the error",
- "type": "object",
- "properties": {
- "crash_address": {
- "type": "string",
- "description": "The relative address in memory were the crash occurred.",
- "examples": [
- "0xabababab"
- ]
- },
- "stacktrace_snippet": {
- "type": "string",
- "description": "The stack trace recorded during fuzzing resulting the crash.",
- "examples": [
- "func_a+0xabcd\nfunc_b+0xabcc"
- ]
- },
- "crash_state": {
- "type": "string",
- "description": "Minimised and normalized crash stack-trace (called crash_state).",
- "examples": [
- "func_a+0xa\nfunc_b+0xb\nfunc_c+0xc"
- ]
- },
- "crash_type": {
- "type": "string",
- "description": "Type of the crash.",
- "examples": [
- "Heap-Buffer-overflow",
- "Division-by-zero"
- ]
- }
- }
- }
- }
- }
- },
- "remediations": {
- "type": "array",
- "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
- "items": {
- "type": "object",
- "required": [
- "fixes",
- "summary",
- "diff"
- ],
- "properties": {
- "fixes": {
- "type": "array",
- "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
- "items": {
- "type": "object",
- "required": [
- "cve"
- ],
- "properties": {
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- }
- }
- }
- },
- "summary": {
- "type": "string",
- "minLength": 1,
- "description": "An overview of how the vulnerabilities were fixed."
- },
- "diff": {
- "type": "string",
- "minLength": 1,
- "description": "A base64-encoded remediation code diff, compatible with git apply."
- }
- }
- }
- }
- }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/dast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/dast-report-format.json
deleted file mode 100644
index 3c9db0546b1..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/dast-report-format.json
+++ /dev/null
@@ -1,1287 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Report format for GitLab DAST",
- "description": "This schema provides the the report format for Dynamic Application Security Testing (https://docs.gitlab.com/ee/user/application_security/dast).",
- "definitions": {
- "detail_type": {
- "oneOf": [
- {
- "$ref": "#/definitions/named_list"
- },
- {
- "$ref": "#/definitions/list"
- },
- {
- "$ref": "#/definitions/table"
- },
- {
- "$ref": "#/definitions/text"
- },
- {
- "$ref": "#/definitions/url"
- },
- {
- "$ref": "#/definitions/code"
- },
- {
- "$ref": "#/definitions/value"
- },
- {
- "$ref": "#/definitions/diff"
- },
- {
- "$ref": "#/definitions/markdown"
- },
- {
- "$ref": "#/definitions/commit"
- },
- {
- "$ref": "#/definitions/file_location"
- },
- {
- "$ref": "#/definitions/module_location"
- }
- ]
- },
- "text_value": {
- "type": "string"
- },
- "named_field": {
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "$ref": "#/definitions/text_value",
- "minLength": 1
- },
- "description": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "named_list": {
- "type": "object",
- "description": "An object with named and typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "named-list"
- },
- "items": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- {
- "$ref": "#/definitions/named_field"
- },
- {
- "$ref": "#/definitions/detail_type"
- }
- ]
- }
- }
- }
- }
- },
- "list": {
- "type": "object",
- "description": "A list of typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- },
- "table": {
- "type": "object",
- "description": "A table of typed fields",
- "required": [
- "type",
- "rows"
- ],
- "properties": {
- "type": {
- "const": "table"
- },
- "header": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- },
- "rows": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- }
- },
- "text": {
- "type": "object",
- "description": "Raw text",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "text"
- },
- "value": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "url": {
- "type": "object",
- "description": "A single URL",
- "required": [
- "type",
- "href"
- ],
- "properties": {
- "type": {
- "const": "url"
- },
- "text": {
- "$ref": "#/definitions/text_value"
- },
- "href": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "http://mysite.com"
- ]
- }
- }
- },
- "code": {
- "type": "object",
- "description": "A codeblock",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "code"
- },
- "value": {
- "type": "string"
- },
- "lang": {
- "type": "string",
- "description": "A programming language"
- }
- }
- },
- "value": {
- "type": "object",
- "description": "A field that can store a range of types of value",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "value"
- },
- "value": {
- "type": [
- "number",
- "string",
- "boolean"
- ]
- }
- }
- },
- "diff": {
- "type": "object",
- "description": "A diff",
- "required": [
- "type",
- "before",
- "after"
- ],
- "properties": {
- "type": {
- "const": "diff"
- },
- "before": {
- "type": "string"
- },
- "after": {
- "type": "string"
- }
- }
- },
- "markdown": {
- "type": "object",
- "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "markdown"
- },
- "value": {
- "$ref": "#/definitions/text_value",
- "examples": [
- "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
- ]
- }
- }
- },
- "commit": {
- "type": "object",
- "description": "A commit/tag/branch within the GitLab project",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "commit"
- },
- "value": {
- "type": "string",
- "description": "The commit SHA",
- "minLength": 1
- }
- }
- },
- "file_location": {
- "type": "object",
- "description": "A location within a file in the project",
- "required": [
- "type",
- "file_name",
- "line_start"
- ],
- "properties": {
- "type": {
- "const": "file-location"
- },
- "file_name": {
- "type": "string",
- "minLength": 1
- },
- "line_start": {
- "type": "integer"
- },
- "line_end": {
- "type": "integer"
- }
- }
- },
- "module_location": {
- "type": "object",
- "description": "A location within a binary module of the form module+relative_offset",
- "required": [
- "type",
- "module_name",
- "offset"
- ],
- "properties": {
- "type": {
- "const": "module-location"
- },
- "module_name": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "compiled_binary"
- ]
- },
- "offset": {
- "type": "integer",
- "examples": [
- 100
- ]
- }
- }
- }
- },
- "self": {
- "version": "14.1.2"
- },
- "required": [
- "version",
- "vulnerabilities"
- ],
- "additionalProperties": true,
- "properties": {
- "scan": {
- "type": "object",
- "required": [
- "end_time",
- "scanned_resources",
- "scanner",
- "start_time",
- "status",
- "type"
- ],
- "properties": {
- "end_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-01-28T03:26:02"
- ]
- },
- "messages": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Communication intended for the initiator of a scan.",
- "required": [
- "level",
- "value"
- ],
- "properties": {
- "level": {
- "type": "string",
- "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
- "enum": [
- "info",
- "warn",
- "fatal"
- ],
- "examples": [
- "info"
- ]
- },
- "value": {
- "type": "string",
- "description": "The message to communicate.",
- "minLength": 1,
- "examples": [
- "Permission denied, scanning aborted"
- ]
- }
- }
- }
- },
- "analyzer": {
- "type": "object",
- "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the analyzer.",
- "minLength": 1,
- "examples": [
- "gitlab-dast"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the analyzer, not required to be unique.",
- "minLength": 1,
- "examples": [
- "GitLab DAST"
- ]
- },
- "url": {
- "type": "string",
- "format": "uri",
- "pattern": "^https?://.+",
- "description": "A link to more information about the analyzer.",
- "examples": [
- "https://docs.gitlab.com/ee/user/application_security/dast"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the analyzer.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- },
- "version": {
- "type": "string",
- "description": "The version of the analyzer.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- }
- }
- },
- "scanner": {
- "type": "object",
- "description": "Object defining the scanner used to perform the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the scanner.",
- "minLength": 1,
- "examples": [
- "my-sast-scanner"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the scanner, not required to be unique.",
- "minLength": 1,
- "examples": [
- "My SAST Scanner"
- ]
- },
- "url": {
- "type": "string",
- "description": "A link to more information about the scanner.",
- "examples": [
- "https://scanner.url"
- ]
- },
- "version": {
- "type": "string",
- "description": "The version of the scanner.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the scanner.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- }
- }
- },
- "start_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-02-14T16:01:59"
- ]
- },
- "status": {
- "type": "string",
- "description": "Result of the scan.",
- "enum": [
- "success",
- "failure"
- ]
- },
- "type": {
- "type": "string",
- "description": "Type of the scan.",
- "enum": [
- "dast",
- "api_fuzzing"
- ]
- },
- "scanned_resources": {
- "type": "array",
- "description": "The attack surface scanned by DAST.",
- "items": {
- "type": "object",
- "required": [
- "method",
- "url",
- "type"
- ],
- "properties": {
- "method": {
- "type": "string",
- "minLength": 1,
- "description": "HTTP method of the scanned resource.",
- "examples": [
- "GET",
- "POST",
- "HEAD"
- ]
- },
- "url": {
- "type": "string",
- "minLength": 1,
- "description": "URL of the scanned resource.",
- "examples": [
- "http://my.site.com/a-page"
- ]
- },
- "type": {
- "type": "string",
- "minLength": 1,
- "description": "Type of the scanned resource, for DAST, this must be 'url'.",
- "examples": [
- "url"
- ]
- }
- }
- }
- }
- }
- },
- "schema": {
- "type": "string",
- "description": "URI pointing to the validating security report schema.",
- "format": "uri"
- },
- "version": {
- "type": "string",
- "description": "The version of the schema to which the JSON report conforms.",
- "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
- },
- "vulnerabilities": {
- "type": "array",
- "description": "Array of vulnerability objects.",
- "items": {
- "type": "object",
- "description": "Describes the vulnerability using GitLab Flavored Markdown",
- "required": [
- "category",
- "cve",
- "identifiers",
- "location",
- "scanner"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
- "examples": [
- "642735a5-1425-428d-8d4e-3c854885a3c9"
- ]
- },
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
- "name": {
- "type": "string",
- "description": "The name of the vulnerability. This must not include the finding's specific information."
- },
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
- "description": {
- "type": "string",
- "description": "A long text section describing the vulnerability more fully."
- },
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
- "severity": {
- "type": "string",
- "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Info",
- "Unknown",
- "Low",
- "Medium",
- "High",
- "Critical"
- ]
- },
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
- "solution": {
- "type": "string",
- "description": "Explanation of how to fix the vulnerability."
- },
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
- "identifiers": {
- "type": "array",
- "minItems": 1,
- "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
- "items": {
- "type": "object",
- "required": [
- "type",
- "name",
- "value"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
- "minLength": 1
- },
- "name": {
- "type": "string",
- "description": "Human-readable name of the identifier.",
- "minLength": 1
- },
- "url": {
- "type": "string",
- "description": "URL of the identifier's documentation.",
- "format": "uri"
- },
- "value": {
- "type": "string",
- "description": "Value of the identifier, for matching purpose.",
- "minLength": 1
- }
- }
- }
- },
- "links": {
- "type": "array",
- "description": "An array of references to external documentation or articles that describe the vulnerability.",
- "items": {
- "type": "object",
- "required": [
- "url"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the vulnerability details link."
- },
- "url": {
- "type": "string",
- "description": "URL of the vulnerability details document.",
- "format": "uri"
- }
- }
- }
- },
- "details": {
- "$ref": "#/definitions/named_list/properties/items"
- },
- "tracking": {
- "description": "Describes how this vulnerability should be tracked as the project changes.",
- "oneOf": [
- {
- "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
- "required": [
- "items"
- ],
- "properties": {
- "type": {
- "const": "source"
- },
- "items": {
- "type": "array",
- "items": {
- "description": "An item that should be tracked using source-specific tracking methods.",
- "type": "object",
- "required": [
- "signatures"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the file that includes the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the file that includes the vulnerability."
- },
- "signatures": {
- "type": "array",
- "description": "An array of calculated tracking signatures for this tracking item.",
- "minItems": 1,
- "items": {
- "description": "A calculated tracking signature value and metadata.",
- "required": [
- "algorithm",
- "value"
- ],
- "properties": {
- "algorithm": {
- "type": "string",
- "description": "The algorithm used to generate the signature."
- },
- "value": {
- "type": "string",
- "description": "The result of this signature algorithm."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "Each tracking type must declare its own type."
- }
- }
- },
- "flags": {
- "description": "Flags that can be attached to vulnerabilities.",
- "type": "array",
- "items": {
- "type": "object",
- "description": "Informational flags identified and assigned to a vulnerability.",
- "required": [
- "type",
- "origin",
- "description"
- ],
- "properties": {
- "type": {
- "type": "string",
- "minLength": 1,
- "description": "Result of the scan.",
- "enum": [
- "flagged-as-likely-false-positive"
- ]
- },
- "origin": {
- "minLength": 1,
- "description": "Tool that issued the flag.",
- "type": "string"
- },
- "description": {
- "minLength": 1,
- "description": "What the flag is about.",
- "type": "string"
- }
- }
- }
- },
- "evidence": {
- "type": "object",
- "properties": {
- "source": {
- "type": "object",
- "description": "Source of evidence",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "Unique source identifier",
- "examples": [
- "assert:LogAnalysis",
- "assert:StatusCode"
- ]
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Source display name",
- "examples": [
- "Log Analysis",
- "Status Code"
- ]
- },
- "url": {
- "type": "string",
- "description": "Link to additional information",
- "examples": [
- "https://docs.gitlab.com/ee/development/integrations/secure.html"
- ]
- }
- }
- },
- "summary": {
- "type": "string",
- "description": "Human readable string containing evidence of the vulnerability.",
- "examples": [
- "Credit card 4111111111111111 found",
- "Server leaked information nginx/1.17.6"
- ]
- },
- "request": {
- "type": "object",
- "description": "An HTTP request.",
- "required": [
- "headers",
- "method",
- "url"
- ],
- "properties": {
- "headers": {
- "type": "array",
- "description": "HTTP headers present on the request.",
- "items": {
- "type": "object",
- "required": [
- "name",
- "value"
- ],
- "properties": {
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Name of the HTTP header.",
- "examples": [
- "Accept",
- "Content-Length",
- "Content-Type"
- ]
- },
- "value": {
- "type": "string",
- "description": "Value of the HTTP header.",
- "examples": [
- "*/*",
- "560",
- "application/json; charset=utf-8"
- ]
- }
- }
- }
- },
- "method": {
- "type": "string",
- "minLength": 1,
- "description": "HTTP method used in the request.",
- "examples": [
- "GET",
- "POST"
- ]
- },
- "url": {
- "type": "string",
- "minLength": 1,
- "description": "URL of the request.",
- "examples": [
- "http://my.site.com/vulnerable-endpoint?show-credit-card"
- ]
- },
- "body": {
- "type": "string",
- "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.",
- "examples": [
- "user=jsmith&first=%27&last=smith"
- ]
- }
- }
- },
- "response": {
- "type": "object",
- "description": "An HTTP response.",
- "required": [
- "headers",
- "reason_phrase",
- "status_code"
- ],
- "properties": {
- "headers": {
- "type": "array",
- "description": "HTTP headers present on the request.",
- "items": {
- "type": "object",
- "required": [
- "name",
- "value"
- ],
- "properties": {
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Name of the HTTP header.",
- "examples": [
- "Accept",
- "Content-Length",
- "Content-Type"
- ]
- },
- "value": {
- "type": "string",
- "description": "Value of the HTTP header.",
- "examples": [
- "*/*",
- "560",
- "application/json; charset=utf-8"
- ]
- }
- }
- }
- },
- "reason_phrase": {
- "type": "string",
- "description": "HTTP reason phrase of the response.",
- "examples": [
- "OK",
- "Internal Server Error"
- ]
- },
- "status_code": {
- "type": "integer",
- "description": "HTTP status code of the response.",
- "examples": [
- 200,
- 500
- ]
- },
- "body": {
- "type": "string",
- "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.",
- "examples": [
- "{\"user_id\": 2}"
- ]
- }
- }
- },
- "supporting_messages": {
- "type": "array",
- "description": "Array of supporting http messages.",
- "items": {
- "type": "object",
- "description": "A supporting http message.",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Message display name.",
- "examples": [
- "Unmodified",
- "Recorded"
- ]
- },
- "request": {
- "type": "object",
- "description": "An HTTP request.",
- "required": [
- "headers",
- "method",
- "url"
- ],
- "properties": {
- "headers": {
- "type": "array",
- "description": "HTTP headers present on the request.",
- "items": {
- "type": "object",
- "required": [
- "name",
- "value"
- ],
- "properties": {
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Name of the HTTP header.",
- "examples": [
- "Accept",
- "Content-Length",
- "Content-Type"
- ]
- },
- "value": {
- "type": "string",
- "description": "Value of the HTTP header.",
- "examples": [
- "*/*",
- "560",
- "application/json; charset=utf-8"
- ]
- }
- }
- }
- },
- "method": {
- "type": "string",
- "minLength": 1,
- "description": "HTTP method used in the request.",
- "examples": [
- "GET",
- "POST"
- ]
- },
- "url": {
- "type": "string",
- "minLength": 1,
- "description": "URL of the request.",
- "examples": [
- "http://my.site.com/vulnerable-endpoint?show-credit-card"
- ]
- },
- "body": {
- "type": "string",
- "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.",
- "examples": [
- "user=jsmith&first=%27&last=smith"
- ]
- }
- }
- },
- "response": {
- "type": "object",
- "description": "An HTTP response.",
- "required": [
- "headers",
- "reason_phrase",
- "status_code"
- ],
- "properties": {
- "headers": {
- "type": "array",
- "description": "HTTP headers present on the request.",
- "items": {
- "type": "object",
- "required": [
- "name",
- "value"
- ],
- "properties": {
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Name of the HTTP header.",
- "examples": [
- "Accept",
- "Content-Length",
- "Content-Type"
- ]
- },
- "value": {
- "type": "string",
- "description": "Value of the HTTP header.",
- "examples": [
- "*/*",
- "560",
- "application/json; charset=utf-8"
- ]
- }
- }
- }
- },
- "reason_phrase": {
- "type": "string",
- "description": "HTTP reason phrase of the response.",
- "examples": [
- "OK",
- "Internal Server Error"
- ]
- },
- "status_code": {
- "type": "integer",
- "description": "HTTP status code of the response.",
- "examples": [
- 200,
- 500
- ]
- },
- "body": {
- "type": "string",
- "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.",
- "examples": [
- "{\"user_id\": 2}"
- ]
- }
- }
- }
- }
- }
- }
- }
- },
- "location": {
- "type": "object",
- "description": "Identifies the vulnerability's location.",
- "properties": {
- "hostname": {
- "type": "string",
- "description": "The protocol, domain, and port of the application where the vulnerability was found."
- },
- "method": {
- "type": "string",
- "description": "The HTTP method that was used to request the URL where the vulnerability was found."
- },
- "param": {
- "type": "string",
- "description": "A value provided by a vulnerability rule related to the found vulnerability. Examples include a header value, or a parameter used in a HTTP POST."
- },
- "path": {
- "type": "string",
- "description": "The path of the URL where the vulnerability was found. Typically, this would start with a forward slash."
- }
- }
- },
- "assets": {
- "type": "array",
- "description": "Array of build assets associated with vulnerability.",
- "items": {
- "type": "object",
- "description": "Describes an asset associated with vulnerability.",
- "required": [
- "type",
- "name",
- "url"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "The type of asset",
- "enum": [
- "http_session",
- "postman"
- ]
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Display name for asset",
- "examples": [
- "HTTP Messages",
- "Postman Collection"
- ]
- },
- "url": {
- "type": "string",
- "minLength": 1,
- "description": "Link to asset in build artifacts",
- "examples": [
- "https://gitlab.com/gitlab-org/security-products/dast/-/jobs/626397001/artifacts/file//output/zap_session.data"
- ]
- }
- }
- }
- },
- "discovered_at": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss.sss, representing when the vulnerability was discovered",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}\\.\\d{3}$",
- "examples": [
- "2020-01-28T03:26:02.956"
- ]
- }
- }
- }
- },
- "remediations": {
- "type": "array",
- "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
- "items": {
- "type": "object",
- "required": [
- "fixes",
- "summary",
- "diff"
- ],
- "properties": {
- "fixes": {
- "type": "array",
- "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
- "items": {
- "type": "object",
- "required": [
- "cve"
- ],
- "properties": {
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- }
- }
- }
- },
- "summary": {
- "type": "string",
- "minLength": 1,
- "description": "An overview of how the vulnerabilities were fixed."
- },
- "diff": {
- "type": "string",
- "minLength": 1,
- "description": "A base64-encoded remediation code diff, compatible with git apply."
- }
- }
- }
- }
- }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/dependency-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/dependency-scanning-report-format.json
deleted file mode 100644
index c7459216faf..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/dependency-scanning-report-format.json
+++ /dev/null
@@ -1,968 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Report format for GitLab Dependency Scanning",
- "description": "This schema provides the the report format for Dependency Scanning analyzers (https://docs.gitlab.com/ee/user/application_security/dependency_scanning).",
- "definitions": {
- "detail_type": {
- "oneOf": [
- {
- "$ref": "#/definitions/named_list"
- },
- {
- "$ref": "#/definitions/list"
- },
- {
- "$ref": "#/definitions/table"
- },
- {
- "$ref": "#/definitions/text"
- },
- {
- "$ref": "#/definitions/url"
- },
- {
- "$ref": "#/definitions/code"
- },
- {
- "$ref": "#/definitions/value"
- },
- {
- "$ref": "#/definitions/diff"
- },
- {
- "$ref": "#/definitions/markdown"
- },
- {
- "$ref": "#/definitions/commit"
- },
- {
- "$ref": "#/definitions/file_location"
- },
- {
- "$ref": "#/definitions/module_location"
- }
- ]
- },
- "text_value": {
- "type": "string"
- },
- "named_field": {
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "$ref": "#/definitions/text_value",
- "minLength": 1
- },
- "description": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "named_list": {
- "type": "object",
- "description": "An object with named and typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "named-list"
- },
- "items": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- {
- "$ref": "#/definitions/named_field"
- },
- {
- "$ref": "#/definitions/detail_type"
- }
- ]
- }
- }
- }
- }
- },
- "list": {
- "type": "object",
- "description": "A list of typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- },
- "table": {
- "type": "object",
- "description": "A table of typed fields",
- "required": [
- "type",
- "rows"
- ],
- "properties": {
- "type": {
- "const": "table"
- },
- "header": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- },
- "rows": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- }
- },
- "text": {
- "type": "object",
- "description": "Raw text",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "text"
- },
- "value": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "url": {
- "type": "object",
- "description": "A single URL",
- "required": [
- "type",
- "href"
- ],
- "properties": {
- "type": {
- "const": "url"
- },
- "text": {
- "$ref": "#/definitions/text_value"
- },
- "href": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "http://mysite.com"
- ]
- }
- }
- },
- "code": {
- "type": "object",
- "description": "A codeblock",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "code"
- },
- "value": {
- "type": "string"
- },
- "lang": {
- "type": "string",
- "description": "A programming language"
- }
- }
- },
- "value": {
- "type": "object",
- "description": "A field that can store a range of types of value",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "value"
- },
- "value": {
- "type": [
- "number",
- "string",
- "boolean"
- ]
- }
- }
- },
- "diff": {
- "type": "object",
- "description": "A diff",
- "required": [
- "type",
- "before",
- "after"
- ],
- "properties": {
- "type": {
- "const": "diff"
- },
- "before": {
- "type": "string"
- },
- "after": {
- "type": "string"
- }
- }
- },
- "markdown": {
- "type": "object",
- "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "markdown"
- },
- "value": {
- "$ref": "#/definitions/text_value",
- "examples": [
- "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
- ]
- }
- }
- },
- "commit": {
- "type": "object",
- "description": "A commit/tag/branch within the GitLab project",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "commit"
- },
- "value": {
- "type": "string",
- "description": "The commit SHA",
- "minLength": 1
- }
- }
- },
- "file_location": {
- "type": "object",
- "description": "A location within a file in the project",
- "required": [
- "type",
- "file_name",
- "line_start"
- ],
- "properties": {
- "type": {
- "const": "file-location"
- },
- "file_name": {
- "type": "string",
- "minLength": 1
- },
- "line_start": {
- "type": "integer"
- },
- "line_end": {
- "type": "integer"
- }
- }
- },
- "module_location": {
- "type": "object",
- "description": "A location within a binary module of the form module+relative_offset",
- "required": [
- "type",
- "module_name",
- "offset"
- ],
- "properties": {
- "type": {
- "const": "module-location"
- },
- "module_name": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "compiled_binary"
- ]
- },
- "offset": {
- "type": "integer",
- "examples": [
- 100
- ]
- }
- }
- }
- },
- "self": {
- "version": "14.1.2"
- },
- "required": [
- "dependency_files",
- "version",
- "vulnerabilities"
- ],
- "additionalProperties": true,
- "properties": {
- "scan": {
- "type": "object",
- "required": [
- "end_time",
- "scanner",
- "start_time",
- "status",
- "type"
- ],
- "properties": {
- "end_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-01-28T03:26:02"
- ]
- },
- "messages": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Communication intended for the initiator of a scan.",
- "required": [
- "level",
- "value"
- ],
- "properties": {
- "level": {
- "type": "string",
- "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
- "enum": [
- "info",
- "warn",
- "fatal"
- ],
- "examples": [
- "info"
- ]
- },
- "value": {
- "type": "string",
- "description": "The message to communicate.",
- "minLength": 1,
- "examples": [
- "Permission denied, scanning aborted"
- ]
- }
- }
- }
- },
- "analyzer": {
- "type": "object",
- "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the analyzer.",
- "minLength": 1,
- "examples": [
- "gitlab-dast"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the analyzer, not required to be unique.",
- "minLength": 1,
- "examples": [
- "GitLab DAST"
- ]
- },
- "url": {
- "type": "string",
- "format": "uri",
- "pattern": "^https?://.+",
- "description": "A link to more information about the analyzer.",
- "examples": [
- "https://docs.gitlab.com/ee/user/application_security/dast"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the analyzer.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- },
- "version": {
- "type": "string",
- "description": "The version of the analyzer.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- }
- }
- },
- "scanner": {
- "type": "object",
- "description": "Object defining the scanner used to perform the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the scanner.",
- "minLength": 1,
- "examples": [
- "my-sast-scanner"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the scanner, not required to be unique.",
- "minLength": 1,
- "examples": [
- "My SAST Scanner"
- ]
- },
- "url": {
- "type": "string",
- "description": "A link to more information about the scanner.",
- "examples": [
- "https://scanner.url"
- ]
- },
- "version": {
- "type": "string",
- "description": "The version of the scanner.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the scanner.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- }
- }
- },
- "start_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-02-14T16:01:59"
- ]
- },
- "status": {
- "type": "string",
- "description": "Result of the scan.",
- "enum": [
- "success",
- "failure"
- ]
- },
- "type": {
- "type": "string",
- "description": "Type of the scan.",
- "enum": [
- "dependency_scanning"
- ]
- }
- }
- },
- "schema": {
- "type": "string",
- "description": "URI pointing to the validating security report schema.",
- "format": "uri"
- },
- "version": {
- "type": "string",
- "description": "The version of the schema to which the JSON report conforms.",
- "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
- },
- "vulnerabilities": {
- "type": "array",
- "description": "Array of vulnerability objects.",
- "items": {
- "type": "object",
- "description": "Describes the vulnerability using GitLab Flavored Markdown",
- "required": [
- "category",
- "cve",
- "identifiers",
- "location",
- "scanner"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
- "examples": [
- "642735a5-1425-428d-8d4e-3c854885a3c9"
- ]
- },
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
- "name": {
- "type": "string",
- "description": "The name of the vulnerability. This must not include the finding's specific information."
- },
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
- "description": {
- "type": "string",
- "description": "A long text section describing the vulnerability more fully."
- },
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
- "severity": {
- "type": "string",
- "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Info",
- "Unknown",
- "Low",
- "Medium",
- "High",
- "Critical"
- ]
- },
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
- "solution": {
- "type": "string",
- "description": "Explanation of how to fix the vulnerability."
- },
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
- "identifiers": {
- "type": "array",
- "minItems": 1,
- "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
- "items": {
- "type": "object",
- "required": [
- "type",
- "name",
- "value"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
- "minLength": 1
- },
- "name": {
- "type": "string",
- "description": "Human-readable name of the identifier.",
- "minLength": 1
- },
- "url": {
- "type": "string",
- "description": "URL of the identifier's documentation.",
- "format": "uri"
- },
- "value": {
- "type": "string",
- "description": "Value of the identifier, for matching purpose.",
- "minLength": 1
- }
- }
- }
- },
- "links": {
- "type": "array",
- "description": "An array of references to external documentation or articles that describe the vulnerability.",
- "items": {
- "type": "object",
- "required": [
- "url"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the vulnerability details link."
- },
- "url": {
- "type": "string",
- "description": "URL of the vulnerability details document.",
- "format": "uri"
- }
- }
- }
- },
- "details": {
- "$ref": "#/definitions/named_list/properties/items"
- },
- "tracking": {
- "description": "Describes how this vulnerability should be tracked as the project changes.",
- "oneOf": [
- {
- "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
- "required": [
- "items"
- ],
- "properties": {
- "type": {
- "const": "source"
- },
- "items": {
- "type": "array",
- "items": {
- "description": "An item that should be tracked using source-specific tracking methods.",
- "type": "object",
- "required": [
- "signatures"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the file that includes the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the file that includes the vulnerability."
- },
- "signatures": {
- "type": "array",
- "description": "An array of calculated tracking signatures for this tracking item.",
- "minItems": 1,
- "items": {
- "description": "A calculated tracking signature value and metadata.",
- "required": [
- "algorithm",
- "value"
- ],
- "properties": {
- "algorithm": {
- "type": "string",
- "description": "The algorithm used to generate the signature."
- },
- "value": {
- "type": "string",
- "description": "The result of this signature algorithm."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "Each tracking type must declare its own type."
- }
- }
- },
- "flags": {
- "description": "Flags that can be attached to vulnerabilities.",
- "type": "array",
- "items": {
- "type": "object",
- "description": "Informational flags identified and assigned to a vulnerability.",
- "required": [
- "type",
- "origin",
- "description"
- ],
- "properties": {
- "type": {
- "type": "string",
- "minLength": 1,
- "description": "Result of the scan.",
- "enum": [
- "flagged-as-likely-false-positive"
- ]
- },
- "origin": {
- "minLength": 1,
- "description": "Tool that issued the flag.",
- "type": "string"
- },
- "description": {
- "minLength": 1,
- "description": "What the flag is about.",
- "type": "string"
- }
- }
- }
- },
- "location": {
- "type": "object",
- "description": "Identifies the vulnerability's location.",
- "required": [
- "file",
- "dependency"
- ],
- "properties": {
- "file": {
- "type": "string",
- "minLength": 1,
- "description": "Path to the manifest or lock file where the dependency is declared (such as yarn.lock)."
- },
- "dependency": {
- "type": "object",
- "description": "Describes the dependency of a project where the vulnerability is located.",
- "properties": {
- "package": {
- "type": "object",
- "description": "Provides information on the package where the vulnerability is located.",
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the package where the vulnerability is located."
- }
- }
- },
- "version": {
- "type": "string",
- "description": "Version of the vulnerable package."
- },
- "iid": {
- "description": "ID that identifies the dependency in the scope of a dependency file.",
- "type": "number"
- },
- "direct": {
- "type": "boolean",
- "description": "Tells whether this is a direct, top-level dependency of the scanned project."
- },
- "dependency_path": {
- "type": "array",
- "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.",
- "items": {
- "type": "object",
- "required": [
- "iid"
- ],
- "properties": {
- "iid": {
- "type": "number",
- "description": "ID that is unique in the scope of a parent object, and specific to the resource type."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- }
- },
- "remediations": {
- "type": "array",
- "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
- "items": {
- "type": "object",
- "required": [
- "fixes",
- "summary",
- "diff"
- ],
- "properties": {
- "fixes": {
- "type": "array",
- "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
- "items": {
- "type": "object",
- "required": [
- "cve"
- ],
- "properties": {
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- }
- }
- }
- },
- "summary": {
- "type": "string",
- "minLength": 1,
- "description": "An overview of how the vulnerabilities were fixed."
- },
- "diff": {
- "type": "string",
- "minLength": 1,
- "description": "A base64-encoded remediation code diff, compatible with git apply."
- }
- }
- }
- },
- "dependency_files": {
- "type": "array",
- "description": "List of dependency files identified in the project.",
- "items": {
- "type": "object",
- "required": [
- "path",
- "package_manager",
- "dependencies"
- ],
- "properties": {
- "path": {
- "type": "string",
- "minLength": 1
- },
- "package_manager": {
- "type": "string",
- "minLength": 1
- },
- "dependencies": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Describes the dependency of a project where the vulnerability is located.",
- "properties": {
- "package": {
- "type": "object",
- "description": "Provides information on the package where the vulnerability is located.",
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the package where the vulnerability is located."
- }
- }
- },
- "version": {
- "type": "string",
- "description": "Version of the vulnerable package."
- },
- "iid": {
- "description": "ID that identifies the dependency in the scope of a dependency file.",
- "type": "number"
- },
- "direct": {
- "type": "boolean",
- "description": "Tells whether this is a direct, top-level dependency of the scanned project."
- },
- "dependency_path": {
- "type": "array",
- "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.",
- "items": {
- "type": "object",
- "required": [
- "iid"
- ],
- "properties": {
- "iid": {
- "type": "number",
- "description": "ID that is unique in the scope of a parent object, and specific to the resource type."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- }
- }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/sast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/sast-report-format.json
deleted file mode 100644
index 20818792652..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/sast-report-format.json
+++ /dev/null
@@ -1,869 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Report format for GitLab SAST",
- "description": "This schema provides the report format for Static Application Security Testing analyzers (https://docs.gitlab.com/ee/user/application_security/sast).",
- "definitions": {
- "detail_type": {
- "oneOf": [
- {
- "$ref": "#/definitions/named_list"
- },
- {
- "$ref": "#/definitions/list"
- },
- {
- "$ref": "#/definitions/table"
- },
- {
- "$ref": "#/definitions/text"
- },
- {
- "$ref": "#/definitions/url"
- },
- {
- "$ref": "#/definitions/code"
- },
- {
- "$ref": "#/definitions/value"
- },
- {
- "$ref": "#/definitions/diff"
- },
- {
- "$ref": "#/definitions/markdown"
- },
- {
- "$ref": "#/definitions/commit"
- },
- {
- "$ref": "#/definitions/file_location"
- },
- {
- "$ref": "#/definitions/module_location"
- }
- ]
- },
- "text_value": {
- "type": "string"
- },
- "named_field": {
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "$ref": "#/definitions/text_value",
- "minLength": 1
- },
- "description": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "named_list": {
- "type": "object",
- "description": "An object with named and typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "named-list"
- },
- "items": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- {
- "$ref": "#/definitions/named_field"
- },
- {
- "$ref": "#/definitions/detail_type"
- }
- ]
- }
- }
- }
- }
- },
- "list": {
- "type": "object",
- "description": "A list of typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- },
- "table": {
- "type": "object",
- "description": "A table of typed fields",
- "required": [
- "type",
- "rows"
- ],
- "properties": {
- "type": {
- "const": "table"
- },
- "header": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- },
- "rows": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- }
- },
- "text": {
- "type": "object",
- "description": "Raw text",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "text"
- },
- "value": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "url": {
- "type": "object",
- "description": "A single URL",
- "required": [
- "type",
- "href"
- ],
- "properties": {
- "type": {
- "const": "url"
- },
- "text": {
- "$ref": "#/definitions/text_value"
- },
- "href": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "http://mysite.com"
- ]
- }
- }
- },
- "code": {
- "type": "object",
- "description": "A codeblock",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "code"
- },
- "value": {
- "type": "string"
- },
- "lang": {
- "type": "string",
- "description": "A programming language"
- }
- }
- },
- "value": {
- "type": "object",
- "description": "A field that can store a range of types of value",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "value"
- },
- "value": {
- "type": [
- "number",
- "string",
- "boolean"
- ]
- }
- }
- },
- "diff": {
- "type": "object",
- "description": "A diff",
- "required": [
- "type",
- "before",
- "after"
- ],
- "properties": {
- "type": {
- "const": "diff"
- },
- "before": {
- "type": "string"
- },
- "after": {
- "type": "string"
- }
- }
- },
- "markdown": {
- "type": "object",
- "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "markdown"
- },
- "value": {
- "$ref": "#/definitions/text_value",
- "examples": [
- "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
- ]
- }
- }
- },
- "commit": {
- "type": "object",
- "description": "A commit/tag/branch within the GitLab project",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "commit"
- },
- "value": {
- "type": "string",
- "description": "The commit SHA",
- "minLength": 1
- }
- }
- },
- "file_location": {
- "type": "object",
- "description": "A location within a file in the project",
- "required": [
- "type",
- "file_name",
- "line_start"
- ],
- "properties": {
- "type": {
- "const": "file-location"
- },
- "file_name": {
- "type": "string",
- "minLength": 1
- },
- "line_start": {
- "type": "integer"
- },
- "line_end": {
- "type": "integer"
- }
- }
- },
- "module_location": {
- "type": "object",
- "description": "A location within a binary module of the form module+relative_offset",
- "required": [
- "type",
- "module_name",
- "offset"
- ],
- "properties": {
- "type": {
- "const": "module-location"
- },
- "module_name": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "compiled_binary"
- ]
- },
- "offset": {
- "type": "integer",
- "examples": [
- 100
- ]
- }
- }
- }
- },
- "self": {
- "version": "14.1.2"
- },
- "required": [
- "version",
- "vulnerabilities"
- ],
- "additionalProperties": true,
- "properties": {
- "scan": {
- "type": "object",
- "required": [
- "end_time",
- "scanner",
- "start_time",
- "status",
- "type"
- ],
- "properties": {
- "end_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-01-28T03:26:02"
- ]
- },
- "messages": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Communication intended for the initiator of a scan.",
- "required": [
- "level",
- "value"
- ],
- "properties": {
- "level": {
- "type": "string",
- "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
- "enum": [
- "info",
- "warn",
- "fatal"
- ],
- "examples": [
- "info"
- ]
- },
- "value": {
- "type": "string",
- "description": "The message to communicate.",
- "minLength": 1,
- "examples": [
- "Permission denied, scanning aborted"
- ]
- }
- }
- }
- },
- "analyzer": {
- "type": "object",
- "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the analyzer.",
- "minLength": 1,
- "examples": [
- "gitlab-dast"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the analyzer, not required to be unique.",
- "minLength": 1,
- "examples": [
- "GitLab DAST"
- ]
- },
- "url": {
- "type": "string",
- "format": "uri",
- "pattern": "^https?://.+",
- "description": "A link to more information about the analyzer.",
- "examples": [
- "https://docs.gitlab.com/ee/user/application_security/dast"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the analyzer.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- },
- "version": {
- "type": "string",
- "description": "The version of the analyzer.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- }
- }
- },
- "scanner": {
- "type": "object",
- "description": "Object defining the scanner used to perform the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the scanner.",
- "minLength": 1,
- "examples": [
- "my-sast-scanner"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the scanner, not required to be unique.",
- "minLength": 1,
- "examples": [
- "My SAST Scanner"
- ]
- },
- "url": {
- "type": "string",
- "description": "A link to more information about the scanner.",
- "examples": [
- "https://scanner.url"
- ]
- },
- "version": {
- "type": "string",
- "description": "The version of the scanner.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the scanner.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- }
- }
- },
- "start_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-02-14T16:01:59"
- ]
- },
- "status": {
- "type": "string",
- "description": "Result of the scan.",
- "enum": [
- "success",
- "failure"
- ]
- },
- "type": {
- "type": "string",
- "description": "Type of the scan.",
- "enum": [
- "sast"
- ]
- }
- }
- },
- "schema": {
- "type": "string",
- "description": "URI pointing to the validating security report schema.",
- "format": "uri"
- },
- "version": {
- "type": "string",
- "description": "The version of the schema to which the JSON report conforms.",
- "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
- },
- "vulnerabilities": {
- "type": "array",
- "description": "Array of vulnerability objects.",
- "items": {
- "type": "object",
- "description": "Describes the vulnerability using GitLab Flavored Markdown",
- "required": [
- "category",
- "cve",
- "identifiers",
- "location",
- "scanner"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
- "examples": [
- "642735a5-1425-428d-8d4e-3c854885a3c9"
- ]
- },
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
- "name": {
- "type": "string",
- "description": "The name of the vulnerability. This must not include the finding's specific information."
- },
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
- "description": {
- "type": "string",
- "description": "A long text section describing the vulnerability more fully."
- },
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
- "severity": {
- "type": "string",
- "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Info",
- "Unknown",
- "Low",
- "Medium",
- "High",
- "Critical"
- ]
- },
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
- "solution": {
- "type": "string",
- "description": "Explanation of how to fix the vulnerability."
- },
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
- "identifiers": {
- "type": "array",
- "minItems": 1,
- "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
- "items": {
- "type": "object",
- "required": [
- "type",
- "name",
- "value"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
- "minLength": 1
- },
- "name": {
- "type": "string",
- "description": "Human-readable name of the identifier.",
- "minLength": 1
- },
- "url": {
- "type": "string",
- "description": "URL of the identifier's documentation.",
- "format": "uri"
- },
- "value": {
- "type": "string",
- "description": "Value of the identifier, for matching purpose.",
- "minLength": 1
- }
- }
- }
- },
- "links": {
- "type": "array",
- "description": "An array of references to external documentation or articles that describe the vulnerability.",
- "items": {
- "type": "object",
- "required": [
- "url"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the vulnerability details link."
- },
- "url": {
- "type": "string",
- "description": "URL of the vulnerability details document.",
- "format": "uri"
- }
- }
- }
- },
- "details": {
- "$ref": "#/definitions/named_list/properties/items"
- },
- "tracking": {
- "description": "Describes how this vulnerability should be tracked as the project changes.",
- "oneOf": [
- {
- "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
- "required": [
- "items"
- ],
- "properties": {
- "type": {
- "const": "source"
- },
- "items": {
- "type": "array",
- "items": {
- "description": "An item that should be tracked using source-specific tracking methods.",
- "type": "object",
- "required": [
- "signatures"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the file that includes the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the file that includes the vulnerability."
- },
- "signatures": {
- "type": "array",
- "description": "An array of calculated tracking signatures for this tracking item.",
- "minItems": 1,
- "items": {
- "description": "A calculated tracking signature value and metadata.",
- "required": [
- "algorithm",
- "value"
- ],
- "properties": {
- "algorithm": {
- "type": "string",
- "description": "The algorithm used to generate the signature."
- },
- "value": {
- "type": "string",
- "description": "The result of this signature algorithm."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "Each tracking type must declare its own type."
- }
- }
- },
- "flags": {
- "description": "Flags that can be attached to vulnerabilities.",
- "type": "array",
- "items": {
- "type": "object",
- "description": "Informational flags identified and assigned to a vulnerability.",
- "required": [
- "type",
- "origin",
- "description"
- ],
- "properties": {
- "type": {
- "type": "string",
- "minLength": 1,
- "description": "Result of the scan.",
- "enum": [
- "flagged-as-likely-false-positive"
- ]
- },
- "origin": {
- "minLength": 1,
- "description": "Tool that issued the flag.",
- "type": "string"
- },
- "description": {
- "minLength": 1,
- "description": "What the flag is about.",
- "type": "string"
- }
- }
- }
- },
- "location": {
- "type": "object",
- "description": "Identifies the vulnerability's location.",
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the code affected by the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the code affected by the vulnerability."
- },
- "class": {
- "type": "string",
- "description": "Provides the name of the class where the vulnerability is located."
- },
- "method": {
- "type": "string",
- "description": "Provides the name of the method where the vulnerability is located."
- }
- }
- },
- "raw_source_code_extract": {
- "type": "string",
- "description": "Provides an unsanitized excerpt of the affected source code."
- }
- }
- }
- },
- "remediations": {
- "type": "array",
- "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
- "items": {
- "type": "object",
- "required": [
- "fixes",
- "summary",
- "diff"
- ],
- "properties": {
- "fixes": {
- "type": "array",
- "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
- "items": {
- "type": "object",
- "required": [
- "cve"
- ],
- "properties": {
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- }
- }
- }
- },
- "summary": {
- "type": "string",
- "minLength": 1,
- "description": "An overview of how the vulnerabilities were fixed."
- },
- "diff": {
- "type": "string",
- "minLength": 1,
- "description": "A base64-encoded remediation code diff, compatible with git apply."
- }
- }
- }
- }
- }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/secret-detection-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/secret-detection-report-format.json
deleted file mode 100644
index 12386d2c1d4..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/secret-detection-report-format.json
+++ /dev/null
@@ -1,892 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Report format for GitLab Secret Detection",
- "description": "This schema provides the the report format for the Secret Detection analyzer (https://docs.gitlab.com/ee/user/application_security/secret_detection)",
- "definitions": {
- "detail_type": {
- "oneOf": [
- {
- "$ref": "#/definitions/named_list"
- },
- {
- "$ref": "#/definitions/list"
- },
- {
- "$ref": "#/definitions/table"
- },
- {
- "$ref": "#/definitions/text"
- },
- {
- "$ref": "#/definitions/url"
- },
- {
- "$ref": "#/definitions/code"
- },
- {
- "$ref": "#/definitions/value"
- },
- {
- "$ref": "#/definitions/diff"
- },
- {
- "$ref": "#/definitions/markdown"
- },
- {
- "$ref": "#/definitions/commit"
- },
- {
- "$ref": "#/definitions/file_location"
- },
- {
- "$ref": "#/definitions/module_location"
- }
- ]
- },
- "text_value": {
- "type": "string"
- },
- "named_field": {
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "$ref": "#/definitions/text_value",
- "minLength": 1
- },
- "description": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "named_list": {
- "type": "object",
- "description": "An object with named and typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "named-list"
- },
- "items": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- {
- "$ref": "#/definitions/named_field"
- },
- {
- "$ref": "#/definitions/detail_type"
- }
- ]
- }
- }
- }
- }
- },
- "list": {
- "type": "object",
- "description": "A list of typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- },
- "table": {
- "type": "object",
- "description": "A table of typed fields",
- "required": [
- "type",
- "rows"
- ],
- "properties": {
- "type": {
- "const": "table"
- },
- "header": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- },
- "rows": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- }
- },
- "text": {
- "type": "object",
- "description": "Raw text",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "text"
- },
- "value": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "url": {
- "type": "object",
- "description": "A single URL",
- "required": [
- "type",
- "href"
- ],
- "properties": {
- "type": {
- "const": "url"
- },
- "text": {
- "$ref": "#/definitions/text_value"
- },
- "href": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "http://mysite.com"
- ]
- }
- }
- },
- "code": {
- "type": "object",
- "description": "A codeblock",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "code"
- },
- "value": {
- "type": "string"
- },
- "lang": {
- "type": "string",
- "description": "A programming language"
- }
- }
- },
- "value": {
- "type": "object",
- "description": "A field that can store a range of types of value",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "value"
- },
- "value": {
- "type": [
- "number",
- "string",
- "boolean"
- ]
- }
- }
- },
- "diff": {
- "type": "object",
- "description": "A diff",
- "required": [
- "type",
- "before",
- "after"
- ],
- "properties": {
- "type": {
- "const": "diff"
- },
- "before": {
- "type": "string"
- },
- "after": {
- "type": "string"
- }
- }
- },
- "markdown": {
- "type": "object",
- "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "markdown"
- },
- "value": {
- "$ref": "#/definitions/text_value",
- "examples": [
- "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
- ]
- }
- }
- },
- "commit": {
- "type": "object",
- "description": "A commit/tag/branch within the GitLab project",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "commit"
- },
- "value": {
- "type": "string",
- "description": "The commit SHA",
- "minLength": 1
- }
- }
- },
- "file_location": {
- "type": "object",
- "description": "A location within a file in the project",
- "required": [
- "type",
- "file_name",
- "line_start"
- ],
- "properties": {
- "type": {
- "const": "file-location"
- },
- "file_name": {
- "type": "string",
- "minLength": 1
- },
- "line_start": {
- "type": "integer"
- },
- "line_end": {
- "type": "integer"
- }
- }
- },
- "module_location": {
- "type": "object",
- "description": "A location within a binary module of the form module+relative_offset",
- "required": [
- "type",
- "module_name",
- "offset"
- ],
- "properties": {
- "type": {
- "const": "module-location"
- },
- "module_name": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "compiled_binary"
- ]
- },
- "offset": {
- "type": "integer",
- "examples": [
- 100
- ]
- }
- }
- }
- },
- "self": {
- "version": "14.1.2"
- },
- "required": [
- "version",
- "vulnerabilities"
- ],
- "additionalProperties": true,
- "properties": {
- "scan": {
- "type": "object",
- "required": [
- "end_time",
- "scanner",
- "start_time",
- "status",
- "type"
- ],
- "properties": {
- "end_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-01-28T03:26:02"
- ]
- },
- "messages": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Communication intended for the initiator of a scan.",
- "required": [
- "level",
- "value"
- ],
- "properties": {
- "level": {
- "type": "string",
- "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
- "enum": [
- "info",
- "warn",
- "fatal"
- ],
- "examples": [
- "info"
- ]
- },
- "value": {
- "type": "string",
- "description": "The message to communicate.",
- "minLength": 1,
- "examples": [
- "Permission denied, scanning aborted"
- ]
- }
- }
- }
- },
- "analyzer": {
- "type": "object",
- "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the analyzer.",
- "minLength": 1,
- "examples": [
- "gitlab-dast"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the analyzer, not required to be unique.",
- "minLength": 1,
- "examples": [
- "GitLab DAST"
- ]
- },
- "url": {
- "type": "string",
- "format": "uri",
- "pattern": "^https?://.+",
- "description": "A link to more information about the analyzer.",
- "examples": [
- "https://docs.gitlab.com/ee/user/application_security/dast"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the analyzer.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- },
- "version": {
- "type": "string",
- "description": "The version of the analyzer.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- }
- }
- },
- "scanner": {
- "type": "object",
- "description": "Object defining the scanner used to perform the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the scanner.",
- "minLength": 1,
- "examples": [
- "my-sast-scanner"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the scanner, not required to be unique.",
- "minLength": 1,
- "examples": [
- "My SAST Scanner"
- ]
- },
- "url": {
- "type": "string",
- "description": "A link to more information about the scanner.",
- "examples": [
- "https://scanner.url"
- ]
- },
- "version": {
- "type": "string",
- "description": "The version of the scanner.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the scanner.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- }
- }
- },
- "start_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-02-14T16:01:59"
- ]
- },
- "status": {
- "type": "string",
- "description": "Result of the scan.",
- "enum": [
- "success",
- "failure"
- ]
- },
- "type": {
- "type": "string",
- "description": "Type of the scan.",
- "enum": [
- "secret_detection"
- ]
- }
- }
- },
- "schema": {
- "type": "string",
- "description": "URI pointing to the validating security report schema.",
- "format": "uri"
- },
- "version": {
- "type": "string",
- "description": "The version of the schema to which the JSON report conforms.",
- "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
- },
- "vulnerabilities": {
- "type": "array",
- "description": "Array of vulnerability objects.",
- "items": {
- "type": "object",
- "description": "Describes the vulnerability using GitLab Flavored Markdown",
- "required": [
- "category",
- "cve",
- "identifiers",
- "location",
- "scanner"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
- "examples": [
- "642735a5-1425-428d-8d4e-3c854885a3c9"
- ]
- },
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
- "name": {
- "type": "string",
- "description": "The name of the vulnerability. This must not include the finding's specific information."
- },
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
- "description": {
- "type": "string",
- "description": "A long text section describing the vulnerability more fully."
- },
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
- "severity": {
- "type": "string",
- "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Info",
- "Unknown",
- "Low",
- "Medium",
- "High",
- "Critical"
- ]
- },
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
- "solution": {
- "type": "string",
- "description": "Explanation of how to fix the vulnerability."
- },
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
- "identifiers": {
- "type": "array",
- "minItems": 1,
- "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
- "items": {
- "type": "object",
- "required": [
- "type",
- "name",
- "value"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
- "minLength": 1
- },
- "name": {
- "type": "string",
- "description": "Human-readable name of the identifier.",
- "minLength": 1
- },
- "url": {
- "type": "string",
- "description": "URL of the identifier's documentation.",
- "format": "uri"
- },
- "value": {
- "type": "string",
- "description": "Value of the identifier, for matching purpose.",
- "minLength": 1
- }
- }
- }
- },
- "links": {
- "type": "array",
- "description": "An array of references to external documentation or articles that describe the vulnerability.",
- "items": {
- "type": "object",
- "required": [
- "url"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the vulnerability details link."
- },
- "url": {
- "type": "string",
- "description": "URL of the vulnerability details document.",
- "format": "uri"
- }
- }
- }
- },
- "details": {
- "$ref": "#/definitions/named_list/properties/items"
- },
- "tracking": {
- "description": "Describes how this vulnerability should be tracked as the project changes.",
- "oneOf": [
- {
- "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
- "required": [
- "items"
- ],
- "properties": {
- "type": {
- "const": "source"
- },
- "items": {
- "type": "array",
- "items": {
- "description": "An item that should be tracked using source-specific tracking methods.",
- "type": "object",
- "required": [
- "signatures"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the file that includes the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the file that includes the vulnerability."
- },
- "signatures": {
- "type": "array",
- "description": "An array of calculated tracking signatures for this tracking item.",
- "minItems": 1,
- "items": {
- "description": "A calculated tracking signature value and metadata.",
- "required": [
- "algorithm",
- "value"
- ],
- "properties": {
- "algorithm": {
- "type": "string",
- "description": "The algorithm used to generate the signature."
- },
- "value": {
- "type": "string",
- "description": "The result of this signature algorithm."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "Each tracking type must declare its own type."
- }
- }
- },
- "flags": {
- "description": "Flags that can be attached to vulnerabilities.",
- "type": "array",
- "items": {
- "type": "object",
- "description": "Informational flags identified and assigned to a vulnerability.",
- "required": [
- "type",
- "origin",
- "description"
- ],
- "properties": {
- "type": {
- "type": "string",
- "minLength": 1,
- "description": "Result of the scan.",
- "enum": [
- "flagged-as-likely-false-positive"
- ]
- },
- "origin": {
- "minLength": 1,
- "description": "Tool that issued the flag.",
- "type": "string"
- },
- "description": {
- "minLength": 1,
- "description": "What the flag is about.",
- "type": "string"
- }
- }
- }
- },
- "location": {
- "required": [
- "commit"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located"
- },
- "commit": {
- "type": "object",
- "description": "Represents the commit in which the vulnerability was detected",
- "required": [
- "sha"
- ],
- "properties": {
- "author": {
- "type": "string"
- },
- "date": {
- "type": "string"
- },
- "message": {
- "type": "string"
- },
- "sha": {
- "type": "string",
- "minLength": 1
- }
- }
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the code affected by the vulnerability"
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the code affected by the vulnerability"
- },
- "class": {
- "type": "string",
- "description": "Provides the name of the class where the vulnerability is located"
- },
- "method": {
- "type": "string",
- "description": "Provides the name of the method where the vulnerability is located"
- }
- }
- },
- "raw_source_code_extract": {
- "type": "string",
- "description": "Provides an unsanitized excerpt of the affected source code."
- }
- }
- }
- },
- "remediations": {
- "type": "array",
- "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
- "items": {
- "type": "object",
- "required": [
- "fixes",
- "summary",
- "diff"
- ],
- "properties": {
- "fixes": {
- "type": "array",
- "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
- "items": {
- "type": "object",
- "required": [
- "cve"
- ],
- "properties": {
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- }
- }
- }
- },
- "summary": {
- "type": "string",
- "minLength": 1,
- "description": "An overview of how the vulnerabilities were fixed."
- },
- "diff": {
- "type": "string",
- "minLength": 1,
- "description": "A base64-encoded remediation code diff, compatible with git apply."
- }
- }
- }
- }
- }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/cluster-image-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/cluster-image-scanning-report-format.json
deleted file mode 100644
index db4c7ab1425..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/cluster-image-scanning-report-format.json
+++ /dev/null
@@ -1,977 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Report format for GitLab Cluster Image Scanning",
- "description": "This schema provides the the report format for Cluster Image Scanning (https://docs.gitlab.com/ee/user/application_security/cluster_image_scanning/).",
- "definitions": {
- "detail_type": {
- "oneOf": [
- {
- "$ref": "#/definitions/named_list"
- },
- {
- "$ref": "#/definitions/list"
- },
- {
- "$ref": "#/definitions/table"
- },
- {
- "$ref": "#/definitions/text"
- },
- {
- "$ref": "#/definitions/url"
- },
- {
- "$ref": "#/definitions/code"
- },
- {
- "$ref": "#/definitions/value"
- },
- {
- "$ref": "#/definitions/diff"
- },
- {
- "$ref": "#/definitions/markdown"
- },
- {
- "$ref": "#/definitions/commit"
- },
- {
- "$ref": "#/definitions/file_location"
- },
- {
- "$ref": "#/definitions/module_location"
- }
- ]
- },
- "text_value": {
- "type": "string"
- },
- "named_field": {
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "$ref": "#/definitions/text_value",
- "minLength": 1
- },
- "description": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "named_list": {
- "type": "object",
- "description": "An object with named and typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "named-list"
- },
- "items": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- {
- "$ref": "#/definitions/named_field"
- },
- {
- "$ref": "#/definitions/detail_type"
- }
- ]
- }
- }
- }
- }
- },
- "list": {
- "type": "object",
- "description": "A list of typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- },
- "table": {
- "type": "object",
- "description": "A table of typed fields",
- "required": [
- "type",
- "rows"
- ],
- "properties": {
- "type": {
- "const": "table"
- },
- "header": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- },
- "rows": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- }
- },
- "text": {
- "type": "object",
- "description": "Raw text",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "text"
- },
- "value": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "url": {
- "type": "object",
- "description": "A single URL",
- "required": [
- "type",
- "href"
- ],
- "properties": {
- "type": {
- "const": "url"
- },
- "text": {
- "$ref": "#/definitions/text_value"
- },
- "href": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "http://mysite.com"
- ]
- }
- }
- },
- "code": {
- "type": "object",
- "description": "A codeblock",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "code"
- },
- "value": {
- "type": "string"
- },
- "lang": {
- "type": "string",
- "description": "A programming language"
- }
- }
- },
- "value": {
- "type": "object",
- "description": "A field that can store a range of types of value",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "value"
- },
- "value": {
- "type": [
- "number",
- "string",
- "boolean"
- ]
- }
- }
- },
- "diff": {
- "type": "object",
- "description": "A diff",
- "required": [
- "type",
- "before",
- "after"
- ],
- "properties": {
- "type": {
- "const": "diff"
- },
- "before": {
- "type": "string"
- },
- "after": {
- "type": "string"
- }
- }
- },
- "markdown": {
- "type": "object",
- "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "markdown"
- },
- "value": {
- "$ref": "#/definitions/text_value",
- "examples": [
- "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
- ]
- }
- }
- },
- "commit": {
- "type": "object",
- "description": "A commit/tag/branch within the GitLab project",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "commit"
- },
- "value": {
- "type": "string",
- "description": "The commit SHA",
- "minLength": 1
- }
- }
- },
- "file_location": {
- "type": "object",
- "description": "A location within a file in the project",
- "required": [
- "type",
- "file_name",
- "line_start"
- ],
- "properties": {
- "type": {
- "const": "file-location"
- },
- "file_name": {
- "type": "string",
- "minLength": 1
- },
- "line_start": {
- "type": "integer"
- },
- "line_end": {
- "type": "integer"
- }
- }
- },
- "module_location": {
- "type": "object",
- "description": "A location within a binary module of the form module+relative_offset",
- "required": [
- "type",
- "module_name",
- "offset"
- ],
- "properties": {
- "type": {
- "const": "module-location"
- },
- "module_name": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "compiled_binary"
- ]
- },
- "offset": {
- "type": "integer",
- "examples": [
- 100
- ]
- }
- }
- }
- },
- "self": {
- "version": "14.1.3"
- },
- "required": [
- "version",
- "vulnerabilities"
- ],
- "additionalProperties": true,
- "properties": {
- "scan": {
- "type": "object",
- "required": [
- "end_time",
- "scanner",
- "start_time",
- "status",
- "type"
- ],
- "properties": {
- "end_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-01-28T03:26:02"
- ]
- },
- "messages": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Communication intended for the initiator of a scan.",
- "required": [
- "level",
- "value"
- ],
- "properties": {
- "level": {
- "type": "string",
- "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
- "enum": [
- "info",
- "warn",
- "fatal"
- ],
- "examples": [
- "info"
- ]
- },
- "value": {
- "type": "string",
- "description": "The message to communicate.",
- "minLength": 1,
- "examples": [
- "Permission denied, scanning aborted"
- ]
- }
- }
- }
- },
- "analyzer": {
- "type": "object",
- "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the analyzer.",
- "minLength": 1,
- "examples": [
- "gitlab-dast"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the analyzer, not required to be unique.",
- "minLength": 1,
- "examples": [
- "GitLab DAST"
- ]
- },
- "url": {
- "type": "string",
- "format": "uri",
- "pattern": "^https?://.+",
- "description": "A link to more information about the analyzer.",
- "examples": [
- "https://docs.gitlab.com/ee/user/application_security/dast"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the analyzer.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- },
- "version": {
- "type": "string",
- "description": "The version of the analyzer.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- }
- }
- },
- "scanner": {
- "type": "object",
- "description": "Object defining the scanner used to perform the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the scanner.",
- "minLength": 1,
- "examples": [
- "my-sast-scanner"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the scanner, not required to be unique.",
- "minLength": 1,
- "examples": [
- "My SAST Scanner"
- ]
- },
- "url": {
- "type": "string",
- "description": "A link to more information about the scanner.",
- "examples": [
- "https://scanner.url"
- ]
- },
- "version": {
- "type": "string",
- "description": "The version of the scanner.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the scanner.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- }
- }
- },
- "start_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-02-14T16:01:59"
- ]
- },
- "status": {
- "type": "string",
- "description": "Result of the scan.",
- "enum": [
- "success",
- "failure"
- ]
- },
- "type": {
- "type": "string",
- "description": "Type of the scan.",
- "enum": [
- "cluster_image_scanning"
- ]
- }
- }
- },
- "schema": {
- "type": "string",
- "description": "URI pointing to the validating security report schema.",
- "format": "uri"
- },
- "version": {
- "type": "string",
- "description": "The version of the schema to which the JSON report conforms.",
- "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
- },
- "vulnerabilities": {
- "type": "array",
- "description": "Array of vulnerability objects.",
- "items": {
- "type": "object",
- "description": "Describes the vulnerability using GitLab Flavored Markdown",
- "required": [
- "category",
- "cve",
- "identifiers",
- "location",
- "scanner"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
- "examples": [
- "642735a5-1425-428d-8d4e-3c854885a3c9"
- ]
- },
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
- "name": {
- "type": "string",
- "description": "The name of the vulnerability. This must not include the finding's specific information."
- },
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
- "description": {
- "type": "string",
- "description": "A long text section describing the vulnerability more fully."
- },
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
- "severity": {
- "type": "string",
- "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Info",
- "Unknown",
- "Low",
- "Medium",
- "High",
- "Critical"
- ]
- },
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
- "solution": {
- "type": "string",
- "description": "Explanation of how to fix the vulnerability."
- },
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
- "identifiers": {
- "type": "array",
- "minItems": 1,
- "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
- "items": {
- "type": "object",
- "required": [
- "type",
- "name",
- "value"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
- "minLength": 1
- },
- "name": {
- "type": "string",
- "description": "Human-readable name of the identifier.",
- "minLength": 1
- },
- "url": {
- "type": "string",
- "description": "URL of the identifier's documentation.",
- "format": "uri"
- },
- "value": {
- "type": "string",
- "description": "Value of the identifier, for matching purpose.",
- "minLength": 1
- }
- }
- }
- },
- "links": {
- "type": "array",
- "description": "An array of references to external documentation or articles that describe the vulnerability.",
- "items": {
- "type": "object",
- "required": [
- "url"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the vulnerability details link."
- },
- "url": {
- "type": "string",
- "description": "URL of the vulnerability details document.",
- "format": "uri"
- }
- }
- }
- },
- "details": {
- "$ref": "#/definitions/named_list/properties/items"
- },
- "tracking": {
- "description": "Describes how this vulnerability should be tracked as the project changes.",
- "oneOf": [
- {
- "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
- "required": [
- "items"
- ],
- "properties": {
- "type": {
- "const": "source"
- },
- "items": {
- "type": "array",
- "items": {
- "description": "An item that should be tracked using source-specific tracking methods.",
- "type": "object",
- "required": [
- "signatures"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the file that includes the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the file that includes the vulnerability."
- },
- "signatures": {
- "type": "array",
- "description": "An array of calculated tracking signatures for this tracking item.",
- "minItems": 1,
- "items": {
- "description": "A calculated tracking signature value and metadata.",
- "required": [
- "algorithm",
- "value"
- ],
- "properties": {
- "algorithm": {
- "type": "string",
- "description": "The algorithm used to generate the signature."
- },
- "value": {
- "type": "string",
- "description": "The result of this signature algorithm."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "Each tracking type must declare its own type."
- }
- }
- },
- "flags": {
- "description": "Flags that can be attached to vulnerabilities.",
- "type": "array",
- "items": {
- "type": "object",
- "description": "Informational flags identified and assigned to a vulnerability.",
- "required": [
- "type",
- "origin",
- "description"
- ],
- "properties": {
- "type": {
- "type": "string",
- "minLength": 1,
- "description": "Result of the scan.",
- "enum": [
- "flagged-as-likely-false-positive"
- ]
- },
- "origin": {
- "minLength": 1,
- "description": "Tool that issued the flag.",
- "type": "string"
- },
- "description": {
- "minLength": 1,
- "description": "What the flag is about.",
- "type": "string"
- }
- }
- }
- },
- "location": {
- "type": "object",
- "description": "Identifies the vulnerability's location.",
- "required": [
- "dependency",
- "image",
- "kubernetes_resource"
- ],
- "properties": {
- "dependency": {
- "type": "object",
- "description": "Describes the dependency of a project where the vulnerability is located.",
- "properties": {
- "package": {
- "type": "object",
- "description": "Provides information on the package where the vulnerability is located.",
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the package where the vulnerability is located."
- }
- }
- },
- "version": {
- "type": "string",
- "description": "Version of the vulnerable package."
- },
- "iid": {
- "description": "ID that identifies the dependency in the scope of a dependency file.",
- "type": "number"
- },
- "direct": {
- "type": "boolean",
- "description": "Tells whether this is a direct, top-level dependency of the scanned project."
- },
- "dependency_path": {
- "type": "array",
- "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.",
- "items": {
- "type": "object",
- "required": [
- "iid"
- ],
- "properties": {
- "iid": {
- "type": "number",
- "description": "ID that is unique in the scope of a parent object, and specific to the resource type."
- }
- }
- }
- }
- }
- },
- "operating_system": {
- "type": "string",
- "minLength": 1,
- "maxLength": 255,
- "description": "The operating system that contains the vulnerable package."
- },
- "image": {
- "type": "string",
- "minLength": 1,
- "description": "The analyzed Docker image.",
- "examples": [
- "index.docker.io/library/nginx:1.21"
- ]
- },
- "kubernetes_resource": {
- "type": "object",
- "description": "The specific Kubernetes resource that was scanned.",
- "required": [
- "namespace",
- "kind",
- "name",
- "container_name"
- ],
- "properties": {
- "namespace": {
- "type": "string",
- "minLength": 1,
- "maxLength": 255,
- "description": "The Kubernetes namespace the resource that had its image scanned.",
- "examples": [
- "default",
- "staging",
- "production"
- ]
- },
- "kind": {
- "type": "string",
- "minLength": 1,
- "maxLength": 255,
- "description": "The Kubernetes kind the resource that had its image scanned.",
- "examples": [
- "Deployment",
- "DaemonSet"
- ]
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "maxLength": 255,
- "description": "The name of the resource that had its image scanned.",
- "examples": [
- "nginx-ingress"
- ]
- },
- "container_name": {
- "type": "string",
- "minLength": 1,
- "maxLength": 255,
- "description": "The name of the container that had its image scanned.",
- "examples": [
- "nginx"
- ]
- },
- "agent_id": {
- "type": "string",
- "minLength": 1,
- "maxLength": 255,
- "description": "The GitLab ID of the Kubernetes Agent which performed the scan.",
- "examples": [
- "1234"
- ]
- },
- "cluster_id": {
- "type": "string",
- "minLength": 1,
- "maxLength": 255,
- "description": "The GitLab ID of the Kubernetes cluster when using cluster integration.",
- "examples": [
- "1234"
- ]
- }
- }
- }
- }
- }
- }
- }
- },
- "remediations": {
- "type": "array",
- "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
- "items": {
- "type": "object",
- "required": [
- "fixes",
- "summary",
- "diff"
- ],
- "properties": {
- "fixes": {
- "type": "array",
- "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
- "items": {
- "type": "object",
- "required": [
- "cve"
- ],
- "properties": {
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- }
- }
- }
- },
- "summary": {
- "type": "string",
- "minLength": 1,
- "description": "An overview of how the vulnerabilities were fixed."
- },
- "diff": {
- "type": "string",
- "minLength": 1,
- "description": "A base64-encoded remediation code diff, compatible with git apply."
- }
- }
- }
- }
- }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/container-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/container-scanning-report-format.json
deleted file mode 100644
index 641cfc82e48..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/container-scanning-report-format.json
+++ /dev/null
@@ -1,911 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Report format for GitLab Container Scanning",
- "description": "This schema provides the the report format for Container Scanning (https://docs.gitlab.com/ee/user/application_security/container_scanning).",
- "definitions": {
- "detail_type": {
- "oneOf": [
- {
- "$ref": "#/definitions/named_list"
- },
- {
- "$ref": "#/definitions/list"
- },
- {
- "$ref": "#/definitions/table"
- },
- {
- "$ref": "#/definitions/text"
- },
- {
- "$ref": "#/definitions/url"
- },
- {
- "$ref": "#/definitions/code"
- },
- {
- "$ref": "#/definitions/value"
- },
- {
- "$ref": "#/definitions/diff"
- },
- {
- "$ref": "#/definitions/markdown"
- },
- {
- "$ref": "#/definitions/commit"
- },
- {
- "$ref": "#/definitions/file_location"
- },
- {
- "$ref": "#/definitions/module_location"
- }
- ]
- },
- "text_value": {
- "type": "string"
- },
- "named_field": {
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "$ref": "#/definitions/text_value",
- "minLength": 1
- },
- "description": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "named_list": {
- "type": "object",
- "description": "An object with named and typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "named-list"
- },
- "items": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- {
- "$ref": "#/definitions/named_field"
- },
- {
- "$ref": "#/definitions/detail_type"
- }
- ]
- }
- }
- }
- }
- },
- "list": {
- "type": "object",
- "description": "A list of typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- },
- "table": {
- "type": "object",
- "description": "A table of typed fields",
- "required": [
- "type",
- "rows"
- ],
- "properties": {
- "type": {
- "const": "table"
- },
- "header": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- },
- "rows": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- }
- },
- "text": {
- "type": "object",
- "description": "Raw text",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "text"
- },
- "value": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "url": {
- "type": "object",
- "description": "A single URL",
- "required": [
- "type",
- "href"
- ],
- "properties": {
- "type": {
- "const": "url"
- },
- "text": {
- "$ref": "#/definitions/text_value"
- },
- "href": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "http://mysite.com"
- ]
- }
- }
- },
- "code": {
- "type": "object",
- "description": "A codeblock",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "code"
- },
- "value": {
- "type": "string"
- },
- "lang": {
- "type": "string",
- "description": "A programming language"
- }
- }
- },
- "value": {
- "type": "object",
- "description": "A field that can store a range of types of value",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "value"
- },
- "value": {
- "type": [
- "number",
- "string",
- "boolean"
- ]
- }
- }
- },
- "diff": {
- "type": "object",
- "description": "A diff",
- "required": [
- "type",
- "before",
- "after"
- ],
- "properties": {
- "type": {
- "const": "diff"
- },
- "before": {
- "type": "string"
- },
- "after": {
- "type": "string"
- }
- }
- },
- "markdown": {
- "type": "object",
- "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "markdown"
- },
- "value": {
- "$ref": "#/definitions/text_value",
- "examples": [
- "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
- ]
- }
- }
- },
- "commit": {
- "type": "object",
- "description": "A commit/tag/branch within the GitLab project",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "commit"
- },
- "value": {
- "type": "string",
- "description": "The commit SHA",
- "minLength": 1
- }
- }
- },
- "file_location": {
- "type": "object",
- "description": "A location within a file in the project",
- "required": [
- "type",
- "file_name",
- "line_start"
- ],
- "properties": {
- "type": {
- "const": "file-location"
- },
- "file_name": {
- "type": "string",
- "minLength": 1
- },
- "line_start": {
- "type": "integer"
- },
- "line_end": {
- "type": "integer"
- }
- }
- },
- "module_location": {
- "type": "object",
- "description": "A location within a binary module of the form module+relative_offset",
- "required": [
- "type",
- "module_name",
- "offset"
- ],
- "properties": {
- "type": {
- "const": "module-location"
- },
- "module_name": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "compiled_binary"
- ]
- },
- "offset": {
- "type": "integer",
- "examples": [
- 100
- ]
- }
- }
- }
- },
- "self": {
- "version": "14.1.3"
- },
- "required": [
- "version",
- "vulnerabilities"
- ],
- "additionalProperties": true,
- "properties": {
- "scan": {
- "type": "object",
- "required": [
- "end_time",
- "scanner",
- "start_time",
- "status",
- "type"
- ],
- "properties": {
- "end_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-01-28T03:26:02"
- ]
- },
- "messages": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Communication intended for the initiator of a scan.",
- "required": [
- "level",
- "value"
- ],
- "properties": {
- "level": {
- "type": "string",
- "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
- "enum": [
- "info",
- "warn",
- "fatal"
- ],
- "examples": [
- "info"
- ]
- },
- "value": {
- "type": "string",
- "description": "The message to communicate.",
- "minLength": 1,
- "examples": [
- "Permission denied, scanning aborted"
- ]
- }
- }
- }
- },
- "analyzer": {
- "type": "object",
- "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the analyzer.",
- "minLength": 1,
- "examples": [
- "gitlab-dast"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the analyzer, not required to be unique.",
- "minLength": 1,
- "examples": [
- "GitLab DAST"
- ]
- },
- "url": {
- "type": "string",
- "format": "uri",
- "pattern": "^https?://.+",
- "description": "A link to more information about the analyzer.",
- "examples": [
- "https://docs.gitlab.com/ee/user/application_security/dast"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the analyzer.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- },
- "version": {
- "type": "string",
- "description": "The version of the analyzer.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- }
- }
- },
- "scanner": {
- "type": "object",
- "description": "Object defining the scanner used to perform the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the scanner.",
- "minLength": 1,
- "examples": [
- "my-sast-scanner"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the scanner, not required to be unique.",
- "minLength": 1,
- "examples": [
- "My SAST Scanner"
- ]
- },
- "url": {
- "type": "string",
- "description": "A link to more information about the scanner.",
- "examples": [
- "https://scanner.url"
- ]
- },
- "version": {
- "type": "string",
- "description": "The version of the scanner.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the scanner.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- }
- }
- },
- "start_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-02-14T16:01:59"
- ]
- },
- "status": {
- "type": "string",
- "description": "Result of the scan.",
- "enum": [
- "success",
- "failure"
- ]
- },
- "type": {
- "type": "string",
- "description": "Type of the scan.",
- "enum": [
- "container_scanning"
- ]
- }
- }
- },
- "schema": {
- "type": "string",
- "description": "URI pointing to the validating security report schema.",
- "format": "uri"
- },
- "version": {
- "type": "string",
- "description": "The version of the schema to which the JSON report conforms.",
- "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
- },
- "vulnerabilities": {
- "type": "array",
- "description": "Array of vulnerability objects.",
- "items": {
- "type": "object",
- "description": "Describes the vulnerability using GitLab Flavored Markdown",
- "required": [
- "category",
- "cve",
- "identifiers",
- "location",
- "scanner"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
- "examples": [
- "642735a5-1425-428d-8d4e-3c854885a3c9"
- ]
- },
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
- "name": {
- "type": "string",
- "description": "The name of the vulnerability. This must not include the finding's specific information."
- },
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
- "description": {
- "type": "string",
- "description": "A long text section describing the vulnerability more fully."
- },
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
- "severity": {
- "type": "string",
- "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Info",
- "Unknown",
- "Low",
- "Medium",
- "High",
- "Critical"
- ]
- },
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
- "solution": {
- "type": "string",
- "description": "Explanation of how to fix the vulnerability."
- },
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
- "identifiers": {
- "type": "array",
- "minItems": 1,
- "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
- "items": {
- "type": "object",
- "required": [
- "type",
- "name",
- "value"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
- "minLength": 1
- },
- "name": {
- "type": "string",
- "description": "Human-readable name of the identifier.",
- "minLength": 1
- },
- "url": {
- "type": "string",
- "description": "URL of the identifier's documentation.",
- "format": "uri"
- },
- "value": {
- "type": "string",
- "description": "Value of the identifier, for matching purpose.",
- "minLength": 1
- }
- }
- }
- },
- "links": {
- "type": "array",
- "description": "An array of references to external documentation or articles that describe the vulnerability.",
- "items": {
- "type": "object",
- "required": [
- "url"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the vulnerability details link."
- },
- "url": {
- "type": "string",
- "description": "URL of the vulnerability details document.",
- "format": "uri"
- }
- }
- }
- },
- "details": {
- "$ref": "#/definitions/named_list/properties/items"
- },
- "tracking": {
- "description": "Describes how this vulnerability should be tracked as the project changes.",
- "oneOf": [
- {
- "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
- "required": [
- "items"
- ],
- "properties": {
- "type": {
- "const": "source"
- },
- "items": {
- "type": "array",
- "items": {
- "description": "An item that should be tracked using source-specific tracking methods.",
- "type": "object",
- "required": [
- "signatures"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the file that includes the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the file that includes the vulnerability."
- },
- "signatures": {
- "type": "array",
- "description": "An array of calculated tracking signatures for this tracking item.",
- "minItems": 1,
- "items": {
- "description": "A calculated tracking signature value and metadata.",
- "required": [
- "algorithm",
- "value"
- ],
- "properties": {
- "algorithm": {
- "type": "string",
- "description": "The algorithm used to generate the signature."
- },
- "value": {
- "type": "string",
- "description": "The result of this signature algorithm."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "Each tracking type must declare its own type."
- }
- }
- },
- "flags": {
- "description": "Flags that can be attached to vulnerabilities.",
- "type": "array",
- "items": {
- "type": "object",
- "description": "Informational flags identified and assigned to a vulnerability.",
- "required": [
- "type",
- "origin",
- "description"
- ],
- "properties": {
- "type": {
- "type": "string",
- "minLength": 1,
- "description": "Result of the scan.",
- "enum": [
- "flagged-as-likely-false-positive"
- ]
- },
- "origin": {
- "minLength": 1,
- "description": "Tool that issued the flag.",
- "type": "string"
- },
- "description": {
- "minLength": 1,
- "description": "What the flag is about.",
- "type": "string"
- }
- }
- }
- },
- "location": {
- "type": "object",
- "description": "Identifies the vulnerability's location.",
- "required": [
- "dependency",
- "operating_system",
- "image"
- ],
- "properties": {
- "dependency": {
- "type": "object",
- "description": "Describes the dependency of a project where the vulnerability is located.",
- "properties": {
- "package": {
- "type": "object",
- "description": "Provides information on the package where the vulnerability is located.",
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the package where the vulnerability is located."
- }
- }
- },
- "version": {
- "type": "string",
- "description": "Version of the vulnerable package."
- },
- "iid": {
- "description": "ID that identifies the dependency in the scope of a dependency file.",
- "type": "number"
- },
- "direct": {
- "type": "boolean",
- "description": "Tells whether this is a direct, top-level dependency of the scanned project."
- },
- "dependency_path": {
- "type": "array",
- "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.",
- "items": {
- "type": "object",
- "required": [
- "iid"
- ],
- "properties": {
- "iid": {
- "type": "number",
- "description": "ID that is unique in the scope of a parent object, and specific to the resource type."
- }
- }
- }
- }
- }
- },
- "operating_system": {
- "type": "string",
- "minLength": 1,
- "description": "The operating system that contains the vulnerable package."
- },
- "image": {
- "type": "string",
- "minLength": 1,
- "pattern": "^[^:]+(:\\d+[^:]*)?:[^:]+(:[^:]+)?$",
- "description": "The analyzed Docker image."
- },
- "default_branch_image": {
- "type": "string",
- "maxLength": 255,
- "pattern": "^[a-zA-Z0-9/_.-]+(:\\d+[a-zA-Z0-9/_.-]*)?:[a-zA-Z0-9_.-]+$",
- "description": "The name of the image on the default branch."
- }
- }
- }
- }
- }
- },
- "remediations": {
- "type": "array",
- "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
- "items": {
- "type": "object",
- "required": [
- "fixes",
- "summary",
- "diff"
- ],
- "properties": {
- "fixes": {
- "type": "array",
- "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
- "items": {
- "type": "object",
- "required": [
- "cve"
- ],
- "properties": {
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- }
- }
- }
- },
- "summary": {
- "type": "string",
- "minLength": 1,
- "description": "An overview of how the vulnerabilities were fixed."
- },
- "diff": {
- "type": "string",
- "minLength": 1,
- "description": "A base64-encoded remediation code diff, compatible with git apply."
- }
- }
- }
- }
- }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/coverage-fuzzing-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/coverage-fuzzing-report-format.json
deleted file mode 100644
index 59aa172444d..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/coverage-fuzzing-report-format.json
+++ /dev/null
@@ -1,874 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Report format for GitLab Fuzz Testing",
- "description": "This schema provides the report format for Coverage Guided Fuzz Testing (https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing).",
- "definitions": {
- "detail_type": {
- "oneOf": [
- {
- "$ref": "#/definitions/named_list"
- },
- {
- "$ref": "#/definitions/list"
- },
- {
- "$ref": "#/definitions/table"
- },
- {
- "$ref": "#/definitions/text"
- },
- {
- "$ref": "#/definitions/url"
- },
- {
- "$ref": "#/definitions/code"
- },
- {
- "$ref": "#/definitions/value"
- },
- {
- "$ref": "#/definitions/diff"
- },
- {
- "$ref": "#/definitions/markdown"
- },
- {
- "$ref": "#/definitions/commit"
- },
- {
- "$ref": "#/definitions/file_location"
- },
- {
- "$ref": "#/definitions/module_location"
- }
- ]
- },
- "text_value": {
- "type": "string"
- },
- "named_field": {
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "$ref": "#/definitions/text_value",
- "minLength": 1
- },
- "description": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "named_list": {
- "type": "object",
- "description": "An object with named and typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "named-list"
- },
- "items": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- {
- "$ref": "#/definitions/named_field"
- },
- {
- "$ref": "#/definitions/detail_type"
- }
- ]
- }
- }
- }
- }
- },
- "list": {
- "type": "object",
- "description": "A list of typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- },
- "table": {
- "type": "object",
- "description": "A table of typed fields",
- "required": [
- "type",
- "rows"
- ],
- "properties": {
- "type": {
- "const": "table"
- },
- "header": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- },
- "rows": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- }
- },
- "text": {
- "type": "object",
- "description": "Raw text",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "text"
- },
- "value": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "url": {
- "type": "object",
- "description": "A single URL",
- "required": [
- "type",
- "href"
- ],
- "properties": {
- "type": {
- "const": "url"
- },
- "text": {
- "$ref": "#/definitions/text_value"
- },
- "href": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "http://mysite.com"
- ]
- }
- }
- },
- "code": {
- "type": "object",
- "description": "A codeblock",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "code"
- },
- "value": {
- "type": "string"
- },
- "lang": {
- "type": "string",
- "description": "A programming language"
- }
- }
- },
- "value": {
- "type": "object",
- "description": "A field that can store a range of types of value",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "value"
- },
- "value": {
- "type": [
- "number",
- "string",
- "boolean"
- ]
- }
- }
- },
- "diff": {
- "type": "object",
- "description": "A diff",
- "required": [
- "type",
- "before",
- "after"
- ],
- "properties": {
- "type": {
- "const": "diff"
- },
- "before": {
- "type": "string"
- },
- "after": {
- "type": "string"
- }
- }
- },
- "markdown": {
- "type": "object",
- "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "markdown"
- },
- "value": {
- "$ref": "#/definitions/text_value",
- "examples": [
- "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
- ]
- }
- }
- },
- "commit": {
- "type": "object",
- "description": "A commit/tag/branch within the GitLab project",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "commit"
- },
- "value": {
- "type": "string",
- "description": "The commit SHA",
- "minLength": 1
- }
- }
- },
- "file_location": {
- "type": "object",
- "description": "A location within a file in the project",
- "required": [
- "type",
- "file_name",
- "line_start"
- ],
- "properties": {
- "type": {
- "const": "file-location"
- },
- "file_name": {
- "type": "string",
- "minLength": 1
- },
- "line_start": {
- "type": "integer"
- },
- "line_end": {
- "type": "integer"
- }
- }
- },
- "module_location": {
- "type": "object",
- "description": "A location within a binary module of the form module+relative_offset",
- "required": [
- "type",
- "module_name",
- "offset"
- ],
- "properties": {
- "type": {
- "const": "module-location"
- },
- "module_name": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "compiled_binary"
- ]
- },
- "offset": {
- "type": "integer",
- "examples": [
- 100
- ]
- }
- }
- }
- },
- "self": {
- "version": "14.1.3"
- },
- "required": [
- "version",
- "vulnerabilities"
- ],
- "additionalProperties": true,
- "properties": {
- "scan": {
- "type": "object",
- "required": [
- "end_time",
- "scanner",
- "start_time",
- "status",
- "type"
- ],
- "properties": {
- "end_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-01-28T03:26:02"
- ]
- },
- "messages": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Communication intended for the initiator of a scan.",
- "required": [
- "level",
- "value"
- ],
- "properties": {
- "level": {
- "type": "string",
- "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
- "enum": [
- "info",
- "warn",
- "fatal"
- ],
- "examples": [
- "info"
- ]
- },
- "value": {
- "type": "string",
- "description": "The message to communicate.",
- "minLength": 1,
- "examples": [
- "Permission denied, scanning aborted"
- ]
- }
- }
- }
- },
- "analyzer": {
- "type": "object",
- "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the analyzer.",
- "minLength": 1,
- "examples": [
- "gitlab-dast"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the analyzer, not required to be unique.",
- "minLength": 1,
- "examples": [
- "GitLab DAST"
- ]
- },
- "url": {
- "type": "string",
- "format": "uri",
- "pattern": "^https?://.+",
- "description": "A link to more information about the analyzer.",
- "examples": [
- "https://docs.gitlab.com/ee/user/application_security/dast"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the analyzer.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- },
- "version": {
- "type": "string",
- "description": "The version of the analyzer.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- }
- }
- },
- "scanner": {
- "type": "object",
- "description": "Object defining the scanner used to perform the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the scanner.",
- "minLength": 1,
- "examples": [
- "my-sast-scanner"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the scanner, not required to be unique.",
- "minLength": 1,
- "examples": [
- "My SAST Scanner"
- ]
- },
- "url": {
- "type": "string",
- "description": "A link to more information about the scanner.",
- "examples": [
- "https://scanner.url"
- ]
- },
- "version": {
- "type": "string",
- "description": "The version of the scanner.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the scanner.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- }
- }
- },
- "start_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-02-14T16:01:59"
- ]
- },
- "status": {
- "type": "string",
- "description": "Result of the scan.",
- "enum": [
- "success",
- "failure"
- ]
- },
- "type": {
- "type": "string",
- "description": "Type of the scan.",
- "enum": [
- "coverage_fuzzing"
- ]
- }
- }
- },
- "schema": {
- "type": "string",
- "description": "URI pointing to the validating security report schema.",
- "format": "uri"
- },
- "version": {
- "type": "string",
- "description": "The version of the schema to which the JSON report conforms.",
- "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
- },
- "vulnerabilities": {
- "type": "array",
- "description": "Array of vulnerability objects.",
- "items": {
- "type": "object",
- "description": "Describes the vulnerability using GitLab Flavored Markdown",
- "required": [
- "category",
- "cve",
- "identifiers",
- "location",
- "scanner"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
- "examples": [
- "642735a5-1425-428d-8d4e-3c854885a3c9"
- ]
- },
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
- "name": {
- "type": "string",
- "description": "The name of the vulnerability. This must not include the finding's specific information."
- },
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
- "description": {
- "type": "string",
- "description": "A long text section describing the vulnerability more fully."
- },
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
- "severity": {
- "type": "string",
- "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Info",
- "Unknown",
- "Low",
- "Medium",
- "High",
- "Critical"
- ]
- },
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
- "solution": {
- "type": "string",
- "description": "Explanation of how to fix the vulnerability."
- },
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
- "identifiers": {
- "type": "array",
- "minItems": 1,
- "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
- "items": {
- "type": "object",
- "required": [
- "type",
- "name",
- "value"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
- "minLength": 1
- },
- "name": {
- "type": "string",
- "description": "Human-readable name of the identifier.",
- "minLength": 1
- },
- "url": {
- "type": "string",
- "description": "URL of the identifier's documentation.",
- "format": "uri"
- },
- "value": {
- "type": "string",
- "description": "Value of the identifier, for matching purpose.",
- "minLength": 1
- }
- }
- }
- },
- "links": {
- "type": "array",
- "description": "An array of references to external documentation or articles that describe the vulnerability.",
- "items": {
- "type": "object",
- "required": [
- "url"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the vulnerability details link."
- },
- "url": {
- "type": "string",
- "description": "URL of the vulnerability details document.",
- "format": "uri"
- }
- }
- }
- },
- "details": {
- "$ref": "#/definitions/named_list/properties/items"
- },
- "tracking": {
- "description": "Describes how this vulnerability should be tracked as the project changes.",
- "oneOf": [
- {
- "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
- "required": [
- "items"
- ],
- "properties": {
- "type": {
- "const": "source"
- },
- "items": {
- "type": "array",
- "items": {
- "description": "An item that should be tracked using source-specific tracking methods.",
- "type": "object",
- "required": [
- "signatures"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the file that includes the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the file that includes the vulnerability."
- },
- "signatures": {
- "type": "array",
- "description": "An array of calculated tracking signatures for this tracking item.",
- "minItems": 1,
- "items": {
- "description": "A calculated tracking signature value and metadata.",
- "required": [
- "algorithm",
- "value"
- ],
- "properties": {
- "algorithm": {
- "type": "string",
- "description": "The algorithm used to generate the signature."
- },
- "value": {
- "type": "string",
- "description": "The result of this signature algorithm."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "Each tracking type must declare its own type."
- }
- }
- },
- "flags": {
- "description": "Flags that can be attached to vulnerabilities.",
- "type": "array",
- "items": {
- "type": "object",
- "description": "Informational flags identified and assigned to a vulnerability.",
- "required": [
- "type",
- "origin",
- "description"
- ],
- "properties": {
- "type": {
- "type": "string",
- "minLength": 1,
- "description": "Result of the scan.",
- "enum": [
- "flagged-as-likely-false-positive"
- ]
- },
- "origin": {
- "minLength": 1,
- "description": "Tool that issued the flag.",
- "type": "string"
- },
- "description": {
- "minLength": 1,
- "description": "What the flag is about.",
- "type": "string"
- }
- }
- }
- },
- "location": {
- "description": "The location of the error",
- "type": "object",
- "properties": {
- "crash_address": {
- "type": "string",
- "description": "The relative address in memory were the crash occurred.",
- "examples": [
- "0xabababab"
- ]
- },
- "stacktrace_snippet": {
- "type": "string",
- "description": "The stack trace recorded during fuzzing resulting the crash.",
- "examples": [
- "func_a+0xabcd\nfunc_b+0xabcc"
- ]
- },
- "crash_state": {
- "type": "string",
- "description": "Minimised and normalized crash stack-trace (called crash_state).",
- "examples": [
- "func_a+0xa\nfunc_b+0xb\nfunc_c+0xc"
- ]
- },
- "crash_type": {
- "type": "string",
- "description": "Type of the crash.",
- "examples": [
- "Heap-Buffer-overflow",
- "Division-by-zero"
- ]
- }
- }
- }
- }
- }
- },
- "remediations": {
- "type": "array",
- "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
- "items": {
- "type": "object",
- "required": [
- "fixes",
- "summary",
- "diff"
- ],
- "properties": {
- "fixes": {
- "type": "array",
- "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
- "items": {
- "type": "object",
- "required": [
- "cve"
- ],
- "properties": {
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- }
- }
- }
- },
- "summary": {
- "type": "string",
- "minLength": 1,
- "description": "An overview of how the vulnerabilities were fixed."
- },
- "diff": {
- "type": "string",
- "minLength": 1,
- "description": "A base64-encoded remediation code diff, compatible with git apply."
- }
- }
- }
- }
- }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/dast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/dast-report-format.json
deleted file mode 100644
index 0e4c866794a..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/dast-report-format.json
+++ /dev/null
@@ -1,1287 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Report format for GitLab DAST",
- "description": "This schema provides the the report format for Dynamic Application Security Testing (https://docs.gitlab.com/ee/user/application_security/dast).",
- "definitions": {
- "detail_type": {
- "oneOf": [
- {
- "$ref": "#/definitions/named_list"
- },
- {
- "$ref": "#/definitions/list"
- },
- {
- "$ref": "#/definitions/table"
- },
- {
- "$ref": "#/definitions/text"
- },
- {
- "$ref": "#/definitions/url"
- },
- {
- "$ref": "#/definitions/code"
- },
- {
- "$ref": "#/definitions/value"
- },
- {
- "$ref": "#/definitions/diff"
- },
- {
- "$ref": "#/definitions/markdown"
- },
- {
- "$ref": "#/definitions/commit"
- },
- {
- "$ref": "#/definitions/file_location"
- },
- {
- "$ref": "#/definitions/module_location"
- }
- ]
- },
- "text_value": {
- "type": "string"
- },
- "named_field": {
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "$ref": "#/definitions/text_value",
- "minLength": 1
- },
- "description": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "named_list": {
- "type": "object",
- "description": "An object with named and typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "named-list"
- },
- "items": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- {
- "$ref": "#/definitions/named_field"
- },
- {
- "$ref": "#/definitions/detail_type"
- }
- ]
- }
- }
- }
- }
- },
- "list": {
- "type": "object",
- "description": "A list of typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- },
- "table": {
- "type": "object",
- "description": "A table of typed fields",
- "required": [
- "type",
- "rows"
- ],
- "properties": {
- "type": {
- "const": "table"
- },
- "header": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- },
- "rows": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- }
- },
- "text": {
- "type": "object",
- "description": "Raw text",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "text"
- },
- "value": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "url": {
- "type": "object",
- "description": "A single URL",
- "required": [
- "type",
- "href"
- ],
- "properties": {
- "type": {
- "const": "url"
- },
- "text": {
- "$ref": "#/definitions/text_value"
- },
- "href": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "http://mysite.com"
- ]
- }
- }
- },
- "code": {
- "type": "object",
- "description": "A codeblock",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "code"
- },
- "value": {
- "type": "string"
- },
- "lang": {
- "type": "string",
- "description": "A programming language"
- }
- }
- },
- "value": {
- "type": "object",
- "description": "A field that can store a range of types of value",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "value"
- },
- "value": {
- "type": [
- "number",
- "string",
- "boolean"
- ]
- }
- }
- },
- "diff": {
- "type": "object",
- "description": "A diff",
- "required": [
- "type",
- "before",
- "after"
- ],
- "properties": {
- "type": {
- "const": "diff"
- },
- "before": {
- "type": "string"
- },
- "after": {
- "type": "string"
- }
- }
- },
- "markdown": {
- "type": "object",
- "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "markdown"
- },
- "value": {
- "$ref": "#/definitions/text_value",
- "examples": [
- "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
- ]
- }
- }
- },
- "commit": {
- "type": "object",
- "description": "A commit/tag/branch within the GitLab project",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "commit"
- },
- "value": {
- "type": "string",
- "description": "The commit SHA",
- "minLength": 1
- }
- }
- },
- "file_location": {
- "type": "object",
- "description": "A location within a file in the project",
- "required": [
- "type",
- "file_name",
- "line_start"
- ],
- "properties": {
- "type": {
- "const": "file-location"
- },
- "file_name": {
- "type": "string",
- "minLength": 1
- },
- "line_start": {
- "type": "integer"
- },
- "line_end": {
- "type": "integer"
- }
- }
- },
- "module_location": {
- "type": "object",
- "description": "A location within a binary module of the form module+relative_offset",
- "required": [
- "type",
- "module_name",
- "offset"
- ],
- "properties": {
- "type": {
- "const": "module-location"
- },
- "module_name": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "compiled_binary"
- ]
- },
- "offset": {
- "type": "integer",
- "examples": [
- 100
- ]
- }
- }
- }
- },
- "self": {
- "version": "14.1.3"
- },
- "required": [
- "version",
- "vulnerabilities"
- ],
- "additionalProperties": true,
- "properties": {
- "scan": {
- "type": "object",
- "required": [
- "end_time",
- "scanned_resources",
- "scanner",
- "start_time",
- "status",
- "type"
- ],
- "properties": {
- "end_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-01-28T03:26:02"
- ]
- },
- "messages": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Communication intended for the initiator of a scan.",
- "required": [
- "level",
- "value"
- ],
- "properties": {
- "level": {
- "type": "string",
- "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
- "enum": [
- "info",
- "warn",
- "fatal"
- ],
- "examples": [
- "info"
- ]
- },
- "value": {
- "type": "string",
- "description": "The message to communicate.",
- "minLength": 1,
- "examples": [
- "Permission denied, scanning aborted"
- ]
- }
- }
- }
- },
- "analyzer": {
- "type": "object",
- "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the analyzer.",
- "minLength": 1,
- "examples": [
- "gitlab-dast"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the analyzer, not required to be unique.",
- "minLength": 1,
- "examples": [
- "GitLab DAST"
- ]
- },
- "url": {
- "type": "string",
- "format": "uri",
- "pattern": "^https?://.+",
- "description": "A link to more information about the analyzer.",
- "examples": [
- "https://docs.gitlab.com/ee/user/application_security/dast"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the analyzer.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- },
- "version": {
- "type": "string",
- "description": "The version of the analyzer.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- }
- }
- },
- "scanner": {
- "type": "object",
- "description": "Object defining the scanner used to perform the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the scanner.",
- "minLength": 1,
- "examples": [
- "my-sast-scanner"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the scanner, not required to be unique.",
- "minLength": 1,
- "examples": [
- "My SAST Scanner"
- ]
- },
- "url": {
- "type": "string",
- "description": "A link to more information about the scanner.",
- "examples": [
- "https://scanner.url"
- ]
- },
- "version": {
- "type": "string",
- "description": "The version of the scanner.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the scanner.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- }
- }
- },
- "start_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-02-14T16:01:59"
- ]
- },
- "status": {
- "type": "string",
- "description": "Result of the scan.",
- "enum": [
- "success",
- "failure"
- ]
- },
- "type": {
- "type": "string",
- "description": "Type of the scan.",
- "enum": [
- "dast",
- "api_fuzzing"
- ]
- },
- "scanned_resources": {
- "type": "array",
- "description": "The attack surface scanned by DAST.",
- "items": {
- "type": "object",
- "required": [
- "method",
- "url",
- "type"
- ],
- "properties": {
- "method": {
- "type": "string",
- "minLength": 1,
- "description": "HTTP method of the scanned resource.",
- "examples": [
- "GET",
- "POST",
- "HEAD"
- ]
- },
- "url": {
- "type": "string",
- "minLength": 1,
- "description": "URL of the scanned resource.",
- "examples": [
- "http://my.site.com/a-page"
- ]
- },
- "type": {
- "type": "string",
- "minLength": 1,
- "description": "Type of the scanned resource, for DAST, this must be 'url'.",
- "examples": [
- "url"
- ]
- }
- }
- }
- }
- }
- },
- "schema": {
- "type": "string",
- "description": "URI pointing to the validating security report schema.",
- "format": "uri"
- },
- "version": {
- "type": "string",
- "description": "The version of the schema to which the JSON report conforms.",
- "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
- },
- "vulnerabilities": {
- "type": "array",
- "description": "Array of vulnerability objects.",
- "items": {
- "type": "object",
- "description": "Describes the vulnerability using GitLab Flavored Markdown",
- "required": [
- "category",
- "cve",
- "identifiers",
- "location",
- "scanner"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
- "examples": [
- "642735a5-1425-428d-8d4e-3c854885a3c9"
- ]
- },
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
- "name": {
- "type": "string",
- "description": "The name of the vulnerability. This must not include the finding's specific information."
- },
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
- "description": {
- "type": "string",
- "description": "A long text section describing the vulnerability more fully."
- },
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
- "severity": {
- "type": "string",
- "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Info",
- "Unknown",
- "Low",
- "Medium",
- "High",
- "Critical"
- ]
- },
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
- "solution": {
- "type": "string",
- "description": "Explanation of how to fix the vulnerability."
- },
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
- "identifiers": {
- "type": "array",
- "minItems": 1,
- "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
- "items": {
- "type": "object",
- "required": [
- "type",
- "name",
- "value"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
- "minLength": 1
- },
- "name": {
- "type": "string",
- "description": "Human-readable name of the identifier.",
- "minLength": 1
- },
- "url": {
- "type": "string",
- "description": "URL of the identifier's documentation.",
- "format": "uri"
- },
- "value": {
- "type": "string",
- "description": "Value of the identifier, for matching purpose.",
- "minLength": 1
- }
- }
- }
- },
- "links": {
- "type": "array",
- "description": "An array of references to external documentation or articles that describe the vulnerability.",
- "items": {
- "type": "object",
- "required": [
- "url"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the vulnerability details link."
- },
- "url": {
- "type": "string",
- "description": "URL of the vulnerability details document.",
- "format": "uri"
- }
- }
- }
- },
- "details": {
- "$ref": "#/definitions/named_list/properties/items"
- },
- "tracking": {
- "description": "Describes how this vulnerability should be tracked as the project changes.",
- "oneOf": [
- {
- "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
- "required": [
- "items"
- ],
- "properties": {
- "type": {
- "const": "source"
- },
- "items": {
- "type": "array",
- "items": {
- "description": "An item that should be tracked using source-specific tracking methods.",
- "type": "object",
- "required": [
- "signatures"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the file that includes the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the file that includes the vulnerability."
- },
- "signatures": {
- "type": "array",
- "description": "An array of calculated tracking signatures for this tracking item.",
- "minItems": 1,
- "items": {
- "description": "A calculated tracking signature value and metadata.",
- "required": [
- "algorithm",
- "value"
- ],
- "properties": {
- "algorithm": {
- "type": "string",
- "description": "The algorithm used to generate the signature."
- },
- "value": {
- "type": "string",
- "description": "The result of this signature algorithm."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "Each tracking type must declare its own type."
- }
- }
- },
- "flags": {
- "description": "Flags that can be attached to vulnerabilities.",
- "type": "array",
- "items": {
- "type": "object",
- "description": "Informational flags identified and assigned to a vulnerability.",
- "required": [
- "type",
- "origin",
- "description"
- ],
- "properties": {
- "type": {
- "type": "string",
- "minLength": 1,
- "description": "Result of the scan.",
- "enum": [
- "flagged-as-likely-false-positive"
- ]
- },
- "origin": {
- "minLength": 1,
- "description": "Tool that issued the flag.",
- "type": "string"
- },
- "description": {
- "minLength": 1,
- "description": "What the flag is about.",
- "type": "string"
- }
- }
- }
- },
- "evidence": {
- "type": "object",
- "properties": {
- "source": {
- "type": "object",
- "description": "Source of evidence",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "Unique source identifier",
- "examples": [
- "assert:LogAnalysis",
- "assert:StatusCode"
- ]
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Source display name",
- "examples": [
- "Log Analysis",
- "Status Code"
- ]
- },
- "url": {
- "type": "string",
- "description": "Link to additional information",
- "examples": [
- "https://docs.gitlab.com/ee/development/integrations/secure.html"
- ]
- }
- }
- },
- "summary": {
- "type": "string",
- "description": "Human readable string containing evidence of the vulnerability.",
- "examples": [
- "Credit card 4111111111111111 found",
- "Server leaked information nginx/1.17.6"
- ]
- },
- "request": {
- "type": "object",
- "description": "An HTTP request.",
- "required": [
- "headers",
- "method",
- "url"
- ],
- "properties": {
- "headers": {
- "type": "array",
- "description": "HTTP headers present on the request.",
- "items": {
- "type": "object",
- "required": [
- "name",
- "value"
- ],
- "properties": {
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Name of the HTTP header.",
- "examples": [
- "Accept",
- "Content-Length",
- "Content-Type"
- ]
- },
- "value": {
- "type": "string",
- "description": "Value of the HTTP header.",
- "examples": [
- "*/*",
- "560",
- "application/json; charset=utf-8"
- ]
- }
- }
- }
- },
- "method": {
- "type": "string",
- "minLength": 1,
- "description": "HTTP method used in the request.",
- "examples": [
- "GET",
- "POST"
- ]
- },
- "url": {
- "type": "string",
- "minLength": 1,
- "description": "URL of the request.",
- "examples": [
- "http://my.site.com/vulnerable-endpoint?show-credit-card"
- ]
- },
- "body": {
- "type": "string",
- "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.",
- "examples": [
- "user=jsmith&first=%27&last=smith"
- ]
- }
- }
- },
- "response": {
- "type": "object",
- "description": "An HTTP response.",
- "required": [
- "headers",
- "reason_phrase",
- "status_code"
- ],
- "properties": {
- "headers": {
- "type": "array",
- "description": "HTTP headers present on the request.",
- "items": {
- "type": "object",
- "required": [
- "name",
- "value"
- ],
- "properties": {
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Name of the HTTP header.",
- "examples": [
- "Accept",
- "Content-Length",
- "Content-Type"
- ]
- },
- "value": {
- "type": "string",
- "description": "Value of the HTTP header.",
- "examples": [
- "*/*",
- "560",
- "application/json; charset=utf-8"
- ]
- }
- }
- }
- },
- "reason_phrase": {
- "type": "string",
- "description": "HTTP reason phrase of the response.",
- "examples": [
- "OK",
- "Internal Server Error"
- ]
- },
- "status_code": {
- "type": "integer",
- "description": "HTTP status code of the response.",
- "examples": [
- 200,
- 500
- ]
- },
- "body": {
- "type": "string",
- "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.",
- "examples": [
- "{\"user_id\": 2}"
- ]
- }
- }
- },
- "supporting_messages": {
- "type": "array",
- "description": "Array of supporting http messages.",
- "items": {
- "type": "object",
- "description": "A supporting http message.",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Message display name.",
- "examples": [
- "Unmodified",
- "Recorded"
- ]
- },
- "request": {
- "type": "object",
- "description": "An HTTP request.",
- "required": [
- "headers",
- "method",
- "url"
- ],
- "properties": {
- "headers": {
- "type": "array",
- "description": "HTTP headers present on the request.",
- "items": {
- "type": "object",
- "required": [
- "name",
- "value"
- ],
- "properties": {
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Name of the HTTP header.",
- "examples": [
- "Accept",
- "Content-Length",
- "Content-Type"
- ]
- },
- "value": {
- "type": "string",
- "description": "Value of the HTTP header.",
- "examples": [
- "*/*",
- "560",
- "application/json; charset=utf-8"
- ]
- }
- }
- }
- },
- "method": {
- "type": "string",
- "minLength": 1,
- "description": "HTTP method used in the request.",
- "examples": [
- "GET",
- "POST"
- ]
- },
- "url": {
- "type": "string",
- "minLength": 1,
- "description": "URL of the request.",
- "examples": [
- "http://my.site.com/vulnerable-endpoint?show-credit-card"
- ]
- },
- "body": {
- "type": "string",
- "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.",
- "examples": [
- "user=jsmith&first=%27&last=smith"
- ]
- }
- }
- },
- "response": {
- "type": "object",
- "description": "An HTTP response.",
- "required": [
- "headers",
- "reason_phrase",
- "status_code"
- ],
- "properties": {
- "headers": {
- "type": "array",
- "description": "HTTP headers present on the request.",
- "items": {
- "type": "object",
- "required": [
- "name",
- "value"
- ],
- "properties": {
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Name of the HTTP header.",
- "examples": [
- "Accept",
- "Content-Length",
- "Content-Type"
- ]
- },
- "value": {
- "type": "string",
- "description": "Value of the HTTP header.",
- "examples": [
- "*/*",
- "560",
- "application/json; charset=utf-8"
- ]
- }
- }
- }
- },
- "reason_phrase": {
- "type": "string",
- "description": "HTTP reason phrase of the response.",
- "examples": [
- "OK",
- "Internal Server Error"
- ]
- },
- "status_code": {
- "type": "integer",
- "description": "HTTP status code of the response.",
- "examples": [
- 200,
- 500
- ]
- },
- "body": {
- "type": "string",
- "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.",
- "examples": [
- "{\"user_id\": 2}"
- ]
- }
- }
- }
- }
- }
- }
- }
- },
- "location": {
- "type": "object",
- "description": "Identifies the vulnerability's location.",
- "properties": {
- "hostname": {
- "type": "string",
- "description": "The protocol, domain, and port of the application where the vulnerability was found."
- },
- "method": {
- "type": "string",
- "description": "The HTTP method that was used to request the URL where the vulnerability was found."
- },
- "param": {
- "type": "string",
- "description": "A value provided by a vulnerability rule related to the found vulnerability. Examples include a header value, or a parameter used in a HTTP POST."
- },
- "path": {
- "type": "string",
- "description": "The path of the URL where the vulnerability was found. Typically, this would start with a forward slash."
- }
- }
- },
- "assets": {
- "type": "array",
- "description": "Array of build assets associated with vulnerability.",
- "items": {
- "type": "object",
- "description": "Describes an asset associated with vulnerability.",
- "required": [
- "type",
- "name",
- "url"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "The type of asset",
- "enum": [
- "http_session",
- "postman"
- ]
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Display name for asset",
- "examples": [
- "HTTP Messages",
- "Postman Collection"
- ]
- },
- "url": {
- "type": "string",
- "minLength": 1,
- "description": "Link to asset in build artifacts",
- "examples": [
- "https://gitlab.com/gitlab-org/security-products/dast/-/jobs/626397001/artifacts/file//output/zap_session.data"
- ]
- }
- }
- }
- },
- "discovered_at": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss.sss, representing when the vulnerability was discovered",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}\\.\\d{3}$",
- "examples": [
- "2020-01-28T03:26:02.956"
- ]
- }
- }
- }
- },
- "remediations": {
- "type": "array",
- "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
- "items": {
- "type": "object",
- "required": [
- "fixes",
- "summary",
- "diff"
- ],
- "properties": {
- "fixes": {
- "type": "array",
- "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
- "items": {
- "type": "object",
- "required": [
- "cve"
- ],
- "properties": {
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- }
- }
- }
- },
- "summary": {
- "type": "string",
- "minLength": 1,
- "description": "An overview of how the vulnerabilities were fixed."
- },
- "diff": {
- "type": "string",
- "minLength": 1,
- "description": "A base64-encoded remediation code diff, compatible with git apply."
- }
- }
- }
- }
- }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/dependency-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/dependency-scanning-report-format.json
deleted file mode 100644
index 652c2f48fe4..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/dependency-scanning-report-format.json
+++ /dev/null
@@ -1,968 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Report format for GitLab Dependency Scanning",
- "description": "This schema provides the the report format for Dependency Scanning analyzers (https://docs.gitlab.com/ee/user/application_security/dependency_scanning).",
- "definitions": {
- "detail_type": {
- "oneOf": [
- {
- "$ref": "#/definitions/named_list"
- },
- {
- "$ref": "#/definitions/list"
- },
- {
- "$ref": "#/definitions/table"
- },
- {
- "$ref": "#/definitions/text"
- },
- {
- "$ref": "#/definitions/url"
- },
- {
- "$ref": "#/definitions/code"
- },
- {
- "$ref": "#/definitions/value"
- },
- {
- "$ref": "#/definitions/diff"
- },
- {
- "$ref": "#/definitions/markdown"
- },
- {
- "$ref": "#/definitions/commit"
- },
- {
- "$ref": "#/definitions/file_location"
- },
- {
- "$ref": "#/definitions/module_location"
- }
- ]
- },
- "text_value": {
- "type": "string"
- },
- "named_field": {
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "$ref": "#/definitions/text_value",
- "minLength": 1
- },
- "description": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "named_list": {
- "type": "object",
- "description": "An object with named and typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "named-list"
- },
- "items": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- {
- "$ref": "#/definitions/named_field"
- },
- {
- "$ref": "#/definitions/detail_type"
- }
- ]
- }
- }
- }
- }
- },
- "list": {
- "type": "object",
- "description": "A list of typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- },
- "table": {
- "type": "object",
- "description": "A table of typed fields",
- "required": [
- "type",
- "rows"
- ],
- "properties": {
- "type": {
- "const": "table"
- },
- "header": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- },
- "rows": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- }
- },
- "text": {
- "type": "object",
- "description": "Raw text",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "text"
- },
- "value": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "url": {
- "type": "object",
- "description": "A single URL",
- "required": [
- "type",
- "href"
- ],
- "properties": {
- "type": {
- "const": "url"
- },
- "text": {
- "$ref": "#/definitions/text_value"
- },
- "href": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "http://mysite.com"
- ]
- }
- }
- },
- "code": {
- "type": "object",
- "description": "A codeblock",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "code"
- },
- "value": {
- "type": "string"
- },
- "lang": {
- "type": "string",
- "description": "A programming language"
- }
- }
- },
- "value": {
- "type": "object",
- "description": "A field that can store a range of types of value",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "value"
- },
- "value": {
- "type": [
- "number",
- "string",
- "boolean"
- ]
- }
- }
- },
- "diff": {
- "type": "object",
- "description": "A diff",
- "required": [
- "type",
- "before",
- "after"
- ],
- "properties": {
- "type": {
- "const": "diff"
- },
- "before": {
- "type": "string"
- },
- "after": {
- "type": "string"
- }
- }
- },
- "markdown": {
- "type": "object",
- "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "markdown"
- },
- "value": {
- "$ref": "#/definitions/text_value",
- "examples": [
- "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
- ]
- }
- }
- },
- "commit": {
- "type": "object",
- "description": "A commit/tag/branch within the GitLab project",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "commit"
- },
- "value": {
- "type": "string",
- "description": "The commit SHA",
- "minLength": 1
- }
- }
- },
- "file_location": {
- "type": "object",
- "description": "A location within a file in the project",
- "required": [
- "type",
- "file_name",
- "line_start"
- ],
- "properties": {
- "type": {
- "const": "file-location"
- },
- "file_name": {
- "type": "string",
- "minLength": 1
- },
- "line_start": {
- "type": "integer"
- },
- "line_end": {
- "type": "integer"
- }
- }
- },
- "module_location": {
- "type": "object",
- "description": "A location within a binary module of the form module+relative_offset",
- "required": [
- "type",
- "module_name",
- "offset"
- ],
- "properties": {
- "type": {
- "const": "module-location"
- },
- "module_name": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "compiled_binary"
- ]
- },
- "offset": {
- "type": "integer",
- "examples": [
- 100
- ]
- }
- }
- }
- },
- "self": {
- "version": "14.1.3"
- },
- "required": [
- "dependency_files",
- "version",
- "vulnerabilities"
- ],
- "additionalProperties": true,
- "properties": {
- "scan": {
- "type": "object",
- "required": [
- "end_time",
- "scanner",
- "start_time",
- "status",
- "type"
- ],
- "properties": {
- "end_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-01-28T03:26:02"
- ]
- },
- "messages": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Communication intended for the initiator of a scan.",
- "required": [
- "level",
- "value"
- ],
- "properties": {
- "level": {
- "type": "string",
- "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
- "enum": [
- "info",
- "warn",
- "fatal"
- ],
- "examples": [
- "info"
- ]
- },
- "value": {
- "type": "string",
- "description": "The message to communicate.",
- "minLength": 1,
- "examples": [
- "Permission denied, scanning aborted"
- ]
- }
- }
- }
- },
- "analyzer": {
- "type": "object",
- "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the analyzer.",
- "minLength": 1,
- "examples": [
- "gitlab-dast"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the analyzer, not required to be unique.",
- "minLength": 1,
- "examples": [
- "GitLab DAST"
- ]
- },
- "url": {
- "type": "string",
- "format": "uri",
- "pattern": "^https?://.+",
- "description": "A link to more information about the analyzer.",
- "examples": [
- "https://docs.gitlab.com/ee/user/application_security/dast"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the analyzer.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- },
- "version": {
- "type": "string",
- "description": "The version of the analyzer.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- }
- }
- },
- "scanner": {
- "type": "object",
- "description": "Object defining the scanner used to perform the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the scanner.",
- "minLength": 1,
- "examples": [
- "my-sast-scanner"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the scanner, not required to be unique.",
- "minLength": 1,
- "examples": [
- "My SAST Scanner"
- ]
- },
- "url": {
- "type": "string",
- "description": "A link to more information about the scanner.",
- "examples": [
- "https://scanner.url"
- ]
- },
- "version": {
- "type": "string",
- "description": "The version of the scanner.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the scanner.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- }
- }
- },
- "start_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-02-14T16:01:59"
- ]
- },
- "status": {
- "type": "string",
- "description": "Result of the scan.",
- "enum": [
- "success",
- "failure"
- ]
- },
- "type": {
- "type": "string",
- "description": "Type of the scan.",
- "enum": [
- "dependency_scanning"
- ]
- }
- }
- },
- "schema": {
- "type": "string",
- "description": "URI pointing to the validating security report schema.",
- "format": "uri"
- },
- "version": {
- "type": "string",
- "description": "The version of the schema to which the JSON report conforms.",
- "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
- },
- "vulnerabilities": {
- "type": "array",
- "description": "Array of vulnerability objects.",
- "items": {
- "type": "object",
- "description": "Describes the vulnerability using GitLab Flavored Markdown",
- "required": [
- "category",
- "cve",
- "identifiers",
- "location",
- "scanner"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
- "examples": [
- "642735a5-1425-428d-8d4e-3c854885a3c9"
- ]
- },
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
- "name": {
- "type": "string",
- "description": "The name of the vulnerability. This must not include the finding's specific information."
- },
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
- "description": {
- "type": "string",
- "description": "A long text section describing the vulnerability more fully."
- },
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
- "severity": {
- "type": "string",
- "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Info",
- "Unknown",
- "Low",
- "Medium",
- "High",
- "Critical"
- ]
- },
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
- "solution": {
- "type": "string",
- "description": "Explanation of how to fix the vulnerability."
- },
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
- "identifiers": {
- "type": "array",
- "minItems": 1,
- "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
- "items": {
- "type": "object",
- "required": [
- "type",
- "name",
- "value"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
- "minLength": 1
- },
- "name": {
- "type": "string",
- "description": "Human-readable name of the identifier.",
- "minLength": 1
- },
- "url": {
- "type": "string",
- "description": "URL of the identifier's documentation.",
- "format": "uri"
- },
- "value": {
- "type": "string",
- "description": "Value of the identifier, for matching purpose.",
- "minLength": 1
- }
- }
- }
- },
- "links": {
- "type": "array",
- "description": "An array of references to external documentation or articles that describe the vulnerability.",
- "items": {
- "type": "object",
- "required": [
- "url"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the vulnerability details link."
- },
- "url": {
- "type": "string",
- "description": "URL of the vulnerability details document.",
- "format": "uri"
- }
- }
- }
- },
- "details": {
- "$ref": "#/definitions/named_list/properties/items"
- },
- "tracking": {
- "description": "Describes how this vulnerability should be tracked as the project changes.",
- "oneOf": [
- {
- "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
- "required": [
- "items"
- ],
- "properties": {
- "type": {
- "const": "source"
- },
- "items": {
- "type": "array",
- "items": {
- "description": "An item that should be tracked using source-specific tracking methods.",
- "type": "object",
- "required": [
- "signatures"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the file that includes the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the file that includes the vulnerability."
- },
- "signatures": {
- "type": "array",
- "description": "An array of calculated tracking signatures for this tracking item.",
- "minItems": 1,
- "items": {
- "description": "A calculated tracking signature value and metadata.",
- "required": [
- "algorithm",
- "value"
- ],
- "properties": {
- "algorithm": {
- "type": "string",
- "description": "The algorithm used to generate the signature."
- },
- "value": {
- "type": "string",
- "description": "The result of this signature algorithm."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "Each tracking type must declare its own type."
- }
- }
- },
- "flags": {
- "description": "Flags that can be attached to vulnerabilities.",
- "type": "array",
- "items": {
- "type": "object",
- "description": "Informational flags identified and assigned to a vulnerability.",
- "required": [
- "type",
- "origin",
- "description"
- ],
- "properties": {
- "type": {
- "type": "string",
- "minLength": 1,
- "description": "Result of the scan.",
- "enum": [
- "flagged-as-likely-false-positive"
- ]
- },
- "origin": {
- "minLength": 1,
- "description": "Tool that issued the flag.",
- "type": "string"
- },
- "description": {
- "minLength": 1,
- "description": "What the flag is about.",
- "type": "string"
- }
- }
- }
- },
- "location": {
- "type": "object",
- "description": "Identifies the vulnerability's location.",
- "required": [
- "file",
- "dependency"
- ],
- "properties": {
- "file": {
- "type": "string",
- "minLength": 1,
- "description": "Path to the manifest or lock file where the dependency is declared (such as yarn.lock)."
- },
- "dependency": {
- "type": "object",
- "description": "Describes the dependency of a project where the vulnerability is located.",
- "properties": {
- "package": {
- "type": "object",
- "description": "Provides information on the package where the vulnerability is located.",
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the package where the vulnerability is located."
- }
- }
- },
- "version": {
- "type": "string",
- "description": "Version of the vulnerable package."
- },
- "iid": {
- "description": "ID that identifies the dependency in the scope of a dependency file.",
- "type": "number"
- },
- "direct": {
- "type": "boolean",
- "description": "Tells whether this is a direct, top-level dependency of the scanned project."
- },
- "dependency_path": {
- "type": "array",
- "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.",
- "items": {
- "type": "object",
- "required": [
- "iid"
- ],
- "properties": {
- "iid": {
- "type": "number",
- "description": "ID that is unique in the scope of a parent object, and specific to the resource type."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- }
- },
- "remediations": {
- "type": "array",
- "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
- "items": {
- "type": "object",
- "required": [
- "fixes",
- "summary",
- "diff"
- ],
- "properties": {
- "fixes": {
- "type": "array",
- "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
- "items": {
- "type": "object",
- "required": [
- "cve"
- ],
- "properties": {
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- }
- }
- }
- },
- "summary": {
- "type": "string",
- "minLength": 1,
- "description": "An overview of how the vulnerabilities were fixed."
- },
- "diff": {
- "type": "string",
- "minLength": 1,
- "description": "A base64-encoded remediation code diff, compatible with git apply."
- }
- }
- }
- },
- "dependency_files": {
- "type": "array",
- "description": "List of dependency files identified in the project.",
- "items": {
- "type": "object",
- "required": [
- "path",
- "package_manager",
- "dependencies"
- ],
- "properties": {
- "path": {
- "type": "string",
- "minLength": 1
- },
- "package_manager": {
- "type": "string",
- "minLength": 1
- },
- "dependencies": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Describes the dependency of a project where the vulnerability is located.",
- "properties": {
- "package": {
- "type": "object",
- "description": "Provides information on the package where the vulnerability is located.",
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the package where the vulnerability is located."
- }
- }
- },
- "version": {
- "type": "string",
- "description": "Version of the vulnerable package."
- },
- "iid": {
- "description": "ID that identifies the dependency in the scope of a dependency file.",
- "type": "number"
- },
- "direct": {
- "type": "boolean",
- "description": "Tells whether this is a direct, top-level dependency of the scanned project."
- },
- "dependency_path": {
- "type": "array",
- "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.",
- "items": {
- "type": "object",
- "required": [
- "iid"
- ],
- "properties": {
- "iid": {
- "type": "number",
- "description": "ID that is unique in the scope of a parent object, and specific to the resource type."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- }
- }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/sast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/sast-report-format.json
deleted file mode 100644
index 40d4d9f5287..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/sast-report-format.json
+++ /dev/null
@@ -1,869 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Report format for GitLab SAST",
- "description": "This schema provides the report format for Static Application Security Testing analyzers (https://docs.gitlab.com/ee/user/application_security/sast).",
- "definitions": {
- "detail_type": {
- "oneOf": [
- {
- "$ref": "#/definitions/named_list"
- },
- {
- "$ref": "#/definitions/list"
- },
- {
- "$ref": "#/definitions/table"
- },
- {
- "$ref": "#/definitions/text"
- },
- {
- "$ref": "#/definitions/url"
- },
- {
- "$ref": "#/definitions/code"
- },
- {
- "$ref": "#/definitions/value"
- },
- {
- "$ref": "#/definitions/diff"
- },
- {
- "$ref": "#/definitions/markdown"
- },
- {
- "$ref": "#/definitions/commit"
- },
- {
- "$ref": "#/definitions/file_location"
- },
- {
- "$ref": "#/definitions/module_location"
- }
- ]
- },
- "text_value": {
- "type": "string"
- },
- "named_field": {
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "$ref": "#/definitions/text_value",
- "minLength": 1
- },
- "description": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "named_list": {
- "type": "object",
- "description": "An object with named and typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "named-list"
- },
- "items": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- {
- "$ref": "#/definitions/named_field"
- },
- {
- "$ref": "#/definitions/detail_type"
- }
- ]
- }
- }
- }
- }
- },
- "list": {
- "type": "object",
- "description": "A list of typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- },
- "table": {
- "type": "object",
- "description": "A table of typed fields",
- "required": [
- "type",
- "rows"
- ],
- "properties": {
- "type": {
- "const": "table"
- },
- "header": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- },
- "rows": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- }
- },
- "text": {
- "type": "object",
- "description": "Raw text",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "text"
- },
- "value": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "url": {
- "type": "object",
- "description": "A single URL",
- "required": [
- "type",
- "href"
- ],
- "properties": {
- "type": {
- "const": "url"
- },
- "text": {
- "$ref": "#/definitions/text_value"
- },
- "href": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "http://mysite.com"
- ]
- }
- }
- },
- "code": {
- "type": "object",
- "description": "A codeblock",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "code"
- },
- "value": {
- "type": "string"
- },
- "lang": {
- "type": "string",
- "description": "A programming language"
- }
- }
- },
- "value": {
- "type": "object",
- "description": "A field that can store a range of types of value",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "value"
- },
- "value": {
- "type": [
- "number",
- "string",
- "boolean"
- ]
- }
- }
- },
- "diff": {
- "type": "object",
- "description": "A diff",
- "required": [
- "type",
- "before",
- "after"
- ],
- "properties": {
- "type": {
- "const": "diff"
- },
- "before": {
- "type": "string"
- },
- "after": {
- "type": "string"
- }
- }
- },
- "markdown": {
- "type": "object",
- "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "markdown"
- },
- "value": {
- "$ref": "#/definitions/text_value",
- "examples": [
- "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
- ]
- }
- }
- },
- "commit": {
- "type": "object",
- "description": "A commit/tag/branch within the GitLab project",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "commit"
- },
- "value": {
- "type": "string",
- "description": "The commit SHA",
- "minLength": 1
- }
- }
- },
- "file_location": {
- "type": "object",
- "description": "A location within a file in the project",
- "required": [
- "type",
- "file_name",
- "line_start"
- ],
- "properties": {
- "type": {
- "const": "file-location"
- },
- "file_name": {
- "type": "string",
- "minLength": 1
- },
- "line_start": {
- "type": "integer"
- },
- "line_end": {
- "type": "integer"
- }
- }
- },
- "module_location": {
- "type": "object",
- "description": "A location within a binary module of the form module+relative_offset",
- "required": [
- "type",
- "module_name",
- "offset"
- ],
- "properties": {
- "type": {
- "const": "module-location"
- },
- "module_name": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "compiled_binary"
- ]
- },
- "offset": {
- "type": "integer",
- "examples": [
- 100
- ]
- }
- }
- }
- },
- "self": {
- "version": "14.1.3"
- },
- "required": [
- "version",
- "vulnerabilities"
- ],
- "additionalProperties": true,
- "properties": {
- "scan": {
- "type": "object",
- "required": [
- "end_time",
- "scanner",
- "start_time",
- "status",
- "type"
- ],
- "properties": {
- "end_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-01-28T03:26:02"
- ]
- },
- "messages": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Communication intended for the initiator of a scan.",
- "required": [
- "level",
- "value"
- ],
- "properties": {
- "level": {
- "type": "string",
- "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
- "enum": [
- "info",
- "warn",
- "fatal"
- ],
- "examples": [
- "info"
- ]
- },
- "value": {
- "type": "string",
- "description": "The message to communicate.",
- "minLength": 1,
- "examples": [
- "Permission denied, scanning aborted"
- ]
- }
- }
- }
- },
- "analyzer": {
- "type": "object",
- "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the analyzer.",
- "minLength": 1,
- "examples": [
- "gitlab-dast"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the analyzer, not required to be unique.",
- "minLength": 1,
- "examples": [
- "GitLab DAST"
- ]
- },
- "url": {
- "type": "string",
- "format": "uri",
- "pattern": "^https?://.+",
- "description": "A link to more information about the analyzer.",
- "examples": [
- "https://docs.gitlab.com/ee/user/application_security/dast"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the analyzer.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- },
- "version": {
- "type": "string",
- "description": "The version of the analyzer.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- }
- }
- },
- "scanner": {
- "type": "object",
- "description": "Object defining the scanner used to perform the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the scanner.",
- "minLength": 1,
- "examples": [
- "my-sast-scanner"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the scanner, not required to be unique.",
- "minLength": 1,
- "examples": [
- "My SAST Scanner"
- ]
- },
- "url": {
- "type": "string",
- "description": "A link to more information about the scanner.",
- "examples": [
- "https://scanner.url"
- ]
- },
- "version": {
- "type": "string",
- "description": "The version of the scanner.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the scanner.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- }
- }
- },
- "start_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-02-14T16:01:59"
- ]
- },
- "status": {
- "type": "string",
- "description": "Result of the scan.",
- "enum": [
- "success",
- "failure"
- ]
- },
- "type": {
- "type": "string",
- "description": "Type of the scan.",
- "enum": [
- "sast"
- ]
- }
- }
- },
- "schema": {
- "type": "string",
- "description": "URI pointing to the validating security report schema.",
- "format": "uri"
- },
- "version": {
- "type": "string",
- "description": "The version of the schema to which the JSON report conforms.",
- "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
- },
- "vulnerabilities": {
- "type": "array",
- "description": "Array of vulnerability objects.",
- "items": {
- "type": "object",
- "description": "Describes the vulnerability using GitLab Flavored Markdown",
- "required": [
- "category",
- "cve",
- "identifiers",
- "location",
- "scanner"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
- "examples": [
- "642735a5-1425-428d-8d4e-3c854885a3c9"
- ]
- },
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
- "name": {
- "type": "string",
- "description": "The name of the vulnerability. This must not include the finding's specific information."
- },
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
- "description": {
- "type": "string",
- "description": "A long text section describing the vulnerability more fully."
- },
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
- "severity": {
- "type": "string",
- "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Info",
- "Unknown",
- "Low",
- "Medium",
- "High",
- "Critical"
- ]
- },
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
- "solution": {
- "type": "string",
- "description": "Explanation of how to fix the vulnerability."
- },
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
- "identifiers": {
- "type": "array",
- "minItems": 1,
- "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
- "items": {
- "type": "object",
- "required": [
- "type",
- "name",
- "value"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
- "minLength": 1
- },
- "name": {
- "type": "string",
- "description": "Human-readable name of the identifier.",
- "minLength": 1
- },
- "url": {
- "type": "string",
- "description": "URL of the identifier's documentation.",
- "format": "uri"
- },
- "value": {
- "type": "string",
- "description": "Value of the identifier, for matching purpose.",
- "minLength": 1
- }
- }
- }
- },
- "links": {
- "type": "array",
- "description": "An array of references to external documentation or articles that describe the vulnerability.",
- "items": {
- "type": "object",
- "required": [
- "url"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the vulnerability details link."
- },
- "url": {
- "type": "string",
- "description": "URL of the vulnerability details document.",
- "format": "uri"
- }
- }
- }
- },
- "details": {
- "$ref": "#/definitions/named_list/properties/items"
- },
- "tracking": {
- "description": "Describes how this vulnerability should be tracked as the project changes.",
- "oneOf": [
- {
- "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
- "required": [
- "items"
- ],
- "properties": {
- "type": {
- "const": "source"
- },
- "items": {
- "type": "array",
- "items": {
- "description": "An item that should be tracked using source-specific tracking methods.",
- "type": "object",
- "required": [
- "signatures"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the file that includes the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the file that includes the vulnerability."
- },
- "signatures": {
- "type": "array",
- "description": "An array of calculated tracking signatures for this tracking item.",
- "minItems": 1,
- "items": {
- "description": "A calculated tracking signature value and metadata.",
- "required": [
- "algorithm",
- "value"
- ],
- "properties": {
- "algorithm": {
- "type": "string",
- "description": "The algorithm used to generate the signature."
- },
- "value": {
- "type": "string",
- "description": "The result of this signature algorithm."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "Each tracking type must declare its own type."
- }
- }
- },
- "flags": {
- "description": "Flags that can be attached to vulnerabilities.",
- "type": "array",
- "items": {
- "type": "object",
- "description": "Informational flags identified and assigned to a vulnerability.",
- "required": [
- "type",
- "origin",
- "description"
- ],
- "properties": {
- "type": {
- "type": "string",
- "minLength": 1,
- "description": "Result of the scan.",
- "enum": [
- "flagged-as-likely-false-positive"
- ]
- },
- "origin": {
- "minLength": 1,
- "description": "Tool that issued the flag.",
- "type": "string"
- },
- "description": {
- "minLength": 1,
- "description": "What the flag is about.",
- "type": "string"
- }
- }
- }
- },
- "location": {
- "type": "object",
- "description": "Identifies the vulnerability's location.",
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the code affected by the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the code affected by the vulnerability."
- },
- "class": {
- "type": "string",
- "description": "Provides the name of the class where the vulnerability is located."
- },
- "method": {
- "type": "string",
- "description": "Provides the name of the method where the vulnerability is located."
- }
- }
- },
- "raw_source_code_extract": {
- "type": "string",
- "description": "Provides an unsanitized excerpt of the affected source code."
- }
- }
- }
- },
- "remediations": {
- "type": "array",
- "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
- "items": {
- "type": "object",
- "required": [
- "fixes",
- "summary",
- "diff"
- ],
- "properties": {
- "fixes": {
- "type": "array",
- "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
- "items": {
- "type": "object",
- "required": [
- "cve"
- ],
- "properties": {
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- }
- }
- }
- },
- "summary": {
- "type": "string",
- "minLength": 1,
- "description": "An overview of how the vulnerabilities were fixed."
- },
- "diff": {
- "type": "string",
- "minLength": 1,
- "description": "A base64-encoded remediation code diff, compatible with git apply."
- }
- }
- }
- }
- }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/secret-detection-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/secret-detection-report-format.json
deleted file mode 100644
index cfde126dd7b..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/secret-detection-report-format.json
+++ /dev/null
@@ -1,892 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Report format for GitLab Secret Detection",
- "description": "This schema provides the the report format for the Secret Detection analyzer (https://docs.gitlab.com/ee/user/application_security/secret_detection)",
- "definitions": {
- "detail_type": {
- "oneOf": [
- {
- "$ref": "#/definitions/named_list"
- },
- {
- "$ref": "#/definitions/list"
- },
- {
- "$ref": "#/definitions/table"
- },
- {
- "$ref": "#/definitions/text"
- },
- {
- "$ref": "#/definitions/url"
- },
- {
- "$ref": "#/definitions/code"
- },
- {
- "$ref": "#/definitions/value"
- },
- {
- "$ref": "#/definitions/diff"
- },
- {
- "$ref": "#/definitions/markdown"
- },
- {
- "$ref": "#/definitions/commit"
- },
- {
- "$ref": "#/definitions/file_location"
- },
- {
- "$ref": "#/definitions/module_location"
- }
- ]
- },
- "text_value": {
- "type": "string"
- },
- "named_field": {
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "$ref": "#/definitions/text_value",
- "minLength": 1
- },
- "description": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "named_list": {
- "type": "object",
- "description": "An object with named and typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "named-list"
- },
- "items": {
- "type": "object",
- "patternProperties": {
- "^.*$": {
- "allOf": [
- {
- "$ref": "#/definitions/named_field"
- },
- {
- "$ref": "#/definitions/detail_type"
- }
- ]
- }
- }
- }
- }
- },
- "list": {
- "type": "object",
- "description": "A list of typed fields",
- "required": [
- "type",
- "items"
- ],
- "properties": {
- "type": {
- "const": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- },
- "table": {
- "type": "object",
- "description": "A table of typed fields",
- "required": [
- "type",
- "rows"
- ],
- "properties": {
- "type": {
- "const": "table"
- },
- "header": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- },
- "rows": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/detail_type"
- }
- }
- }
- }
- },
- "text": {
- "type": "object",
- "description": "Raw text",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "text"
- },
- "value": {
- "$ref": "#/definitions/text_value"
- }
- }
- },
- "url": {
- "type": "object",
- "description": "A single URL",
- "required": [
- "type",
- "href"
- ],
- "properties": {
- "type": {
- "const": "url"
- },
- "text": {
- "$ref": "#/definitions/text_value"
- },
- "href": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "http://mysite.com"
- ]
- }
- }
- },
- "code": {
- "type": "object",
- "description": "A codeblock",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "code"
- },
- "value": {
- "type": "string"
- },
- "lang": {
- "type": "string",
- "description": "A programming language"
- }
- }
- },
- "value": {
- "type": "object",
- "description": "A field that can store a range of types of value",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "value"
- },
- "value": {
- "type": [
- "number",
- "string",
- "boolean"
- ]
- }
- }
- },
- "diff": {
- "type": "object",
- "description": "A diff",
- "required": [
- "type",
- "before",
- "after"
- ],
- "properties": {
- "type": {
- "const": "diff"
- },
- "before": {
- "type": "string"
- },
- "after": {
- "type": "string"
- }
- }
- },
- "markdown": {
- "type": "object",
- "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "markdown"
- },
- "value": {
- "$ref": "#/definitions/text_value",
- "examples": [
- "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
- ]
- }
- }
- },
- "commit": {
- "type": "object",
- "description": "A commit/tag/branch within the GitLab project",
- "required": [
- "type",
- "value"
- ],
- "properties": {
- "type": {
- "const": "commit"
- },
- "value": {
- "type": "string",
- "description": "The commit SHA",
- "minLength": 1
- }
- }
- },
- "file_location": {
- "type": "object",
- "description": "A location within a file in the project",
- "required": [
- "type",
- "file_name",
- "line_start"
- ],
- "properties": {
- "type": {
- "const": "file-location"
- },
- "file_name": {
- "type": "string",
- "minLength": 1
- },
- "line_start": {
- "type": "integer"
- },
- "line_end": {
- "type": "integer"
- }
- }
- },
- "module_location": {
- "type": "object",
- "description": "A location within a binary module of the form module+relative_offset",
- "required": [
- "type",
- "module_name",
- "offset"
- ],
- "properties": {
- "type": {
- "const": "module-location"
- },
- "module_name": {
- "type": "string",
- "minLength": 1,
- "examples": [
- "compiled_binary"
- ]
- },
- "offset": {
- "type": "integer",
- "examples": [
- 100
- ]
- }
- }
- }
- },
- "self": {
- "version": "14.1.3"
- },
- "required": [
- "version",
- "vulnerabilities"
- ],
- "additionalProperties": true,
- "properties": {
- "scan": {
- "type": "object",
- "required": [
- "end_time",
- "scanner",
- "start_time",
- "status",
- "type"
- ],
- "properties": {
- "end_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-01-28T03:26:02"
- ]
- },
- "messages": {
- "type": "array",
- "items": {
- "type": "object",
- "description": "Communication intended for the initiator of a scan.",
- "required": [
- "level",
- "value"
- ],
- "properties": {
- "level": {
- "type": "string",
- "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
- "enum": [
- "info",
- "warn",
- "fatal"
- ],
- "examples": [
- "info"
- ]
- },
- "value": {
- "type": "string",
- "description": "The message to communicate.",
- "minLength": 1,
- "examples": [
- "Permission denied, scanning aborted"
- ]
- }
- }
- }
- },
- "analyzer": {
- "type": "object",
- "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the analyzer.",
- "minLength": 1,
- "examples": [
- "gitlab-dast"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the analyzer, not required to be unique.",
- "minLength": 1,
- "examples": [
- "GitLab DAST"
- ]
- },
- "url": {
- "type": "string",
- "format": "uri",
- "pattern": "^https?://.+",
- "description": "A link to more information about the analyzer.",
- "examples": [
- "https://docs.gitlab.com/ee/user/application_security/dast"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the analyzer.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- },
- "version": {
- "type": "string",
- "description": "The version of the analyzer.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- }
- }
- },
- "scanner": {
- "type": "object",
- "description": "Object defining the scanner used to perform the scan.",
- "required": [
- "id",
- "name",
- "version",
- "vendor"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique id that identifies the scanner.",
- "minLength": 1,
- "examples": [
- "my-sast-scanner"
- ]
- },
- "name": {
- "type": "string",
- "description": "A human readable value that identifies the scanner, not required to be unique.",
- "minLength": 1,
- "examples": [
- "My SAST Scanner"
- ]
- },
- "url": {
- "type": "string",
- "description": "A link to more information about the scanner.",
- "examples": [
- "https://scanner.url"
- ]
- },
- "version": {
- "type": "string",
- "description": "The version of the scanner.",
- "minLength": 1,
- "examples": [
- "1.0.2"
- ]
- },
- "vendor": {
- "description": "The vendor/maintainer of the scanner.",
- "type": "object",
- "required": [
- "name"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the vendor.",
- "minLength": 1,
- "examples": [
- "GitLab"
- ]
- }
- }
- }
- }
- },
- "start_time": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
- "examples": [
- "2020-02-14T16:01:59"
- ]
- },
- "status": {
- "type": "string",
- "description": "Result of the scan.",
- "enum": [
- "success",
- "failure"
- ]
- },
- "type": {
- "type": "string",
- "description": "Type of the scan.",
- "enum": [
- "secret_detection"
- ]
- }
- }
- },
- "schema": {
- "type": "string",
- "description": "URI pointing to the validating security report schema.",
- "format": "uri"
- },
- "version": {
- "type": "string",
- "description": "The version of the schema to which the JSON report conforms.",
- "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
- },
- "vulnerabilities": {
- "type": "array",
- "description": "Array of vulnerability objects.",
- "items": {
- "type": "object",
- "description": "Describes the vulnerability using GitLab Flavored Markdown",
- "required": [
- "category",
- "cve",
- "identifiers",
- "location",
- "scanner"
- ],
- "properties": {
- "id": {
- "type": "string",
- "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
- "examples": [
- "642735a5-1425-428d-8d4e-3c854885a3c9"
- ]
- },
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
- "name": {
- "type": "string",
- "description": "The name of the vulnerability. This must not include the finding's specific information."
- },
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
- "description": {
- "type": "string",
- "description": "A long text section describing the vulnerability more fully."
- },
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
- "severity": {
- "type": "string",
- "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Info",
- "Unknown",
- "Low",
- "Medium",
- "High",
- "Critical"
- ]
- },
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
- "solution": {
- "type": "string",
- "description": "Explanation of how to fix the vulnerability."
- },
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
- "identifiers": {
- "type": "array",
- "minItems": 1,
- "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
- "items": {
- "type": "object",
- "required": [
- "type",
- "name",
- "value"
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
- "minLength": 1
- },
- "name": {
- "type": "string",
- "description": "Human-readable name of the identifier.",
- "minLength": 1
- },
- "url": {
- "type": "string",
- "description": "URL of the identifier's documentation.",
- "format": "uri"
- },
- "value": {
- "type": "string",
- "description": "Value of the identifier, for matching purpose.",
- "minLength": 1
- }
- }
- }
- },
- "links": {
- "type": "array",
- "description": "An array of references to external documentation or articles that describe the vulnerability.",
- "items": {
- "type": "object",
- "required": [
- "url"
- ],
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the vulnerability details link."
- },
- "url": {
- "type": "string",
- "description": "URL of the vulnerability details document.",
- "format": "uri"
- }
- }
- }
- },
- "details": {
- "$ref": "#/definitions/named_list/properties/items"
- },
- "tracking": {
- "description": "Describes how this vulnerability should be tracked as the project changes.",
- "oneOf": [
- {
- "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
- "required": [
- "items"
- ],
- "properties": {
- "type": {
- "const": "source"
- },
- "items": {
- "type": "array",
- "items": {
- "description": "An item that should be tracked using source-specific tracking methods.",
- "type": "object",
- "required": [
- "signatures"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located."
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the file that includes the vulnerability."
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the file that includes the vulnerability."
- },
- "signatures": {
- "type": "array",
- "description": "An array of calculated tracking signatures for this tracking item.",
- "minItems": 1,
- "items": {
- "description": "A calculated tracking signature value and metadata.",
- "required": [
- "algorithm",
- "value"
- ],
- "properties": {
- "algorithm": {
- "type": "string",
- "description": "The algorithm used to generate the signature."
- },
- "value": {
- "type": "string",
- "description": "The result of this signature algorithm."
- }
- }
- }
- }
- }
- }
- }
- }
- }
- ],
- "properties": {
- "type": {
- "type": "string",
- "description": "Each tracking type must declare its own type."
- }
- }
- },
- "flags": {
- "description": "Flags that can be attached to vulnerabilities.",
- "type": "array",
- "items": {
- "type": "object",
- "description": "Informational flags identified and assigned to a vulnerability.",
- "required": [
- "type",
- "origin",
- "description"
- ],
- "properties": {
- "type": {
- "type": "string",
- "minLength": 1,
- "description": "Result of the scan.",
- "enum": [
- "flagged-as-likely-false-positive"
- ]
- },
- "origin": {
- "minLength": 1,
- "description": "Tool that issued the flag.",
- "type": "string"
- },
- "description": {
- "minLength": 1,
- "description": "What the flag is about.",
- "type": "string"
- }
- }
- }
- },
- "location": {
- "required": [
- "commit"
- ],
- "properties": {
- "file": {
- "type": "string",
- "description": "Path to the file where the vulnerability is located"
- },
- "commit": {
- "type": "object",
- "description": "Represents the commit in which the vulnerability was detected",
- "required": [
- "sha"
- ],
- "properties": {
- "author": {
- "type": "string"
- },
- "date": {
- "type": "string"
- },
- "message": {
- "type": "string"
- },
- "sha": {
- "type": "string",
- "minLength": 1
- }
- }
- },
- "start_line": {
- "type": "number",
- "description": "The first line of the code affected by the vulnerability"
- },
- "end_line": {
- "type": "number",
- "description": "The last line of the code affected by the vulnerability"
- },
- "class": {
- "type": "string",
- "description": "Provides the name of the class where the vulnerability is located"
- },
- "method": {
- "type": "string",
- "description": "Provides the name of the method where the vulnerability is located"
- }
- }
- },
- "raw_source_code_extract": {
- "type": "string",
- "description": "Provides an unsanitized excerpt of the affected source code."
- }
- }
- }
- },
- "remediations": {
- "type": "array",
- "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
- "items": {
- "type": "object",
- "required": [
- "fixes",
- "summary",
- "diff"
- ],
- "properties": {
- "fixes": {
- "type": "array",
- "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
- "items": {
- "type": "object",
- "required": [
- "cve"
- ],
- "properties": {
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- }
- }
- }
- },
- "summary": {
- "type": "string",
- "minLength": 1,
- "description": "An overview of how the vulnerabilities were fixed."
- },
- "diff": {
- "type": "string",
- "minLength": 1,
- "description": "A base64-encoded remediation code diff, compatible with git apply."
- }
- }
- }
- }
- }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/cluster-image-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.6/cluster-image-scanning-report-format.json
index 7bcb2d5867f..91414255211 100644
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/cluster-image-scanning-report-format.json
+++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.6/cluster-image-scanning-report-format.json
@@ -1,5 +1,6 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
+ "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/cluster-image-scanning-report-format.json",
"title": "Report format for GitLab Cluster Image Scanning",
"description": "This schema provides the the report format for Cluster Image Scanning (https://docs.gitlab.com/ee/user/application_security/cluster_image_scanning/).",
"definitions": {
@@ -54,6 +55,7 @@
"properties": {
"name": {
"$ref": "#/definitions/text_value",
+ "type": "string",
"minLength": 1
},
"description": {
@@ -325,9 +327,11 @@
}
},
"self": {
- "version": "14.1.1"
+ "version": "15.0.6"
},
+ "type": "object",
"required": [
+ "scan",
"version",
"vulnerabilities"
],
@@ -336,6 +340,7 @@
"scan": {
"type": "object",
"required": [
+ "analyzer",
"end_time",
"scanner",
"start_time",
@@ -346,7 +351,7 @@
"end_time": {
"type": "string",
"description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
+ "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$",
"examples": [
"2020-01-28T03:26:02"
]
@@ -384,6 +389,57 @@
}
}
},
+ "options": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "description": "A configuration option used for this scan.",
+ "required": [
+ "name",
+ "value"
+ ],
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "The configuration option name.",
+ "maxLength": 255,
+ "minLength": 1,
+ "examples": [
+ "DAST_FF_ENABLE_BAS",
+ "DOCKER_TLS_CERTDIR",
+ "DS_MAX_DEPTH",
+ "SECURE_LOG_LEVEL"
+ ]
+ },
+ "source": {
+ "type": "string",
+ "description": "The source of this option.",
+ "enum": [
+ "argument",
+ "file",
+ "env_variable",
+ "other"
+ ]
+ },
+ "value": {
+ "type": [
+ "boolean",
+ "integer",
+ "null",
+ "string"
+ ],
+ "description": "The value used for this scan.",
+ "examples": [
+ true,
+ 2,
+ null,
+ "fatal",
+ ""
+ ]
+ }
+ }
+ }
+ },
"analyzer": {
"type": "object",
"description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
@@ -412,7 +468,6 @@
},
"url": {
"type": "string",
- "format": "uri",
"pattern": "^https?://.+",
"description": "A link to more information about the analyzer.",
"examples": [
@@ -509,7 +564,7 @@
"start_time": {
"type": "string",
"description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
+ "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$",
"examples": [
"2020-02-14T16:01:59"
]
@@ -528,13 +583,47 @@
"enum": [
"cluster_image_scanning"
]
+ },
+ "primary_identifiers": {
+ "type": "array",
+ "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results",
+ "items": {
+ "type": "object",
+ "required": [
+ "type",
+ "name",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
+ "minLength": 1
+ },
+ "name": {
+ "type": "string",
+ "description": "Human-readable name of the identifier.",
+ "minLength": 1
+ },
+ "url": {
+ "type": "string",
+ "description": "URL of the identifier's documentation.",
+ "pattern": "^(https?|ftp)://.+"
+ },
+ "value": {
+ "type": "string",
+ "description": "Value of the identifier, for matching purpose.",
+ "minLength": 1
+ }
+ }
+ }
}
}
},
"schema": {
"type": "string",
"description": "URI pointing to the validating security report schema.",
- "format": "uri"
+ "pattern": "^https?://.+"
},
"version": {
"type": "string",
@@ -548,41 +637,29 @@
"type": "object",
"description": "Describes the vulnerability using GitLab Flavored Markdown",
"required": [
- "category",
- "cve",
+ "id",
"identifiers",
- "location",
- "scanner"
+ "location"
],
"properties": {
"id": {
"type": "string",
+ "minLength": 1,
"description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
"examples": [
"642735a5-1425-428d-8d4e-3c854885a3c9"
]
},
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
"name": {
"type": "string",
+ "maxLength": 255,
"description": "The name of the vulnerability. This must not include the finding's specific information."
},
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
"description": {
"type": "string",
+ "maxLength": 1048576,
"description": "A long text section describing the vulnerability more fully."
},
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
"severity": {
"type": "string",
"description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
@@ -595,43 +672,11 @@
"Critical"
]
},
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
"solution": {
"type": "string",
+ "maxLength": 7000,
"description": "Explanation of how to fix the vulnerability."
},
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
"identifiers": {
"type": "array",
"minItems": 1,
@@ -657,7 +702,7 @@
"url": {
"type": "string",
"description": "URL of the identifier's documentation.",
- "format": "uri"
+ "pattern": "^(https?|ftp)://.+"
},
"value": {
"type": "string",
@@ -683,7 +728,7 @@
"url": {
"type": "string",
"description": "URL of the vulnerability details document.",
- "format": "uri"
+ "pattern": "^(https?|ftp)://.+"
}
}
}
@@ -692,6 +737,7 @@
"$ref": "#/definitions/named_list/properties/items"
},
"tracking": {
+ "type": "object",
"description": "Describes how this vulnerability should be tracked as the project changes.",
"oneOf": [
{
@@ -730,6 +776,7 @@
"minItems": 1,
"items": {
"description": "A calculated tracking signature value and metadata.",
+ "type": "object",
"required": [
"algorithm",
"value"
@@ -804,10 +851,17 @@
"dependency": {
"type": "object",
"description": "Describes the dependency of a project where the vulnerability is located.",
+ "required": [
+ "package",
+ "version"
+ ],
"properties": {
"package": {
"type": "object",
"description": "Provides information on the package where the vulnerability is located.",
+ "required": [
+ "name"
+ ],
"properties": {
"name": {
"type": "string",
@@ -950,12 +1004,16 @@
"items": {
"type": "object",
"required": [
- "cve"
+ "id"
],
"properties": {
- "cve": {
+ "id": {
"type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
+ "minLength": 1,
+ "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
+ "examples": [
+ "642735a5-1425-428d-8d4e-3c854885a3c9"
+ ]
}
}
}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.6/container-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.6/container-scanning-report-format.json
index fb412af44e3..ecd92ed2ff1 100644
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.6/container-scanning-report-format.json
+++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.6/container-scanning-report-format.json
@@ -1,5 +1,6 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
+ "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/container-scanning-report-format.json",
"title": "Report format for GitLab Container Scanning",
"description": "This schema provides the the report format for Container Scanning (https://docs.gitlab.com/ee/user/application_security/container_scanning).",
"definitions": {
@@ -54,6 +55,7 @@
"properties": {
"name": {
"$ref": "#/definitions/text_value",
+ "type": "string",
"minLength": 1
},
"description": {
@@ -325,9 +327,11 @@
}
},
"self": {
- "version": "14.0.6"
+ "version": "15.0.6"
},
+ "type": "object",
"required": [
+ "scan",
"version",
"vulnerabilities"
],
@@ -336,6 +340,7 @@
"scan": {
"type": "object",
"required": [
+ "analyzer",
"end_time",
"scanner",
"start_time",
@@ -346,7 +351,7 @@
"end_time": {
"type": "string",
"description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
+ "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$",
"examples": [
"2020-01-28T03:26:02"
]
@@ -384,6 +389,57 @@
}
}
},
+ "options": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "description": "A configuration option used for this scan.",
+ "required": [
+ "name",
+ "value"
+ ],
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "The configuration option name.",
+ "maxLength": 255,
+ "minLength": 1,
+ "examples": [
+ "DAST_FF_ENABLE_BAS",
+ "DOCKER_TLS_CERTDIR",
+ "DS_MAX_DEPTH",
+ "SECURE_LOG_LEVEL"
+ ]
+ },
+ "source": {
+ "type": "string",
+ "description": "The source of this option.",
+ "enum": [
+ "argument",
+ "file",
+ "env_variable",
+ "other"
+ ]
+ },
+ "value": {
+ "type": [
+ "boolean",
+ "integer",
+ "null",
+ "string"
+ ],
+ "description": "The value used for this scan.",
+ "examples": [
+ true,
+ 2,
+ null,
+ "fatal",
+ ""
+ ]
+ }
+ }
+ }
+ },
"analyzer": {
"type": "object",
"description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
@@ -412,7 +468,6 @@
},
"url": {
"type": "string",
- "format": "uri",
"pattern": "^https?://.+",
"description": "A link to more information about the analyzer.",
"examples": [
@@ -509,7 +564,7 @@
"start_time": {
"type": "string",
"description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
+ "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$",
"examples": [
"2020-02-14T16:01:59"
]
@@ -528,13 +583,47 @@
"enum": [
"container_scanning"
]
+ },
+ "primary_identifiers": {
+ "type": "array",
+ "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results",
+ "items": {
+ "type": "object",
+ "required": [
+ "type",
+ "name",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
+ "minLength": 1
+ },
+ "name": {
+ "type": "string",
+ "description": "Human-readable name of the identifier.",
+ "minLength": 1
+ },
+ "url": {
+ "type": "string",
+ "description": "URL of the identifier's documentation.",
+ "pattern": "^(https?|ftp)://.+"
+ },
+ "value": {
+ "type": "string",
+ "description": "Value of the identifier, for matching purpose.",
+ "minLength": 1
+ }
+ }
+ }
}
}
},
"schema": {
"type": "string",
"description": "URI pointing to the validating security report schema.",
- "format": "uri"
+ "pattern": "^https?://.+"
},
"version": {
"type": "string",
@@ -548,41 +637,29 @@
"type": "object",
"description": "Describes the vulnerability using GitLab Flavored Markdown",
"required": [
- "category",
- "cve",
+ "id",
"identifiers",
- "location",
- "scanner"
+ "location"
],
"properties": {
"id": {
"type": "string",
+ "minLength": 1,
"description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
"examples": [
"642735a5-1425-428d-8d4e-3c854885a3c9"
]
},
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
"name": {
"type": "string",
+ "maxLength": 255,
"description": "The name of the vulnerability. This must not include the finding's specific information."
},
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
"description": {
"type": "string",
+ "maxLength": 1048576,
"description": "A long text section describing the vulnerability more fully."
},
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
"severity": {
"type": "string",
"description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
@@ -595,43 +672,11 @@
"Critical"
]
},
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
"solution": {
"type": "string",
+ "maxLength": 7000,
"description": "Explanation of how to fix the vulnerability."
},
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
"identifiers": {
"type": "array",
"minItems": 1,
@@ -657,7 +702,7 @@
"url": {
"type": "string",
"description": "URL of the identifier's documentation.",
- "format": "uri"
+ "pattern": "^(https?|ftp)://.+"
},
"value": {
"type": "string",
@@ -683,7 +728,7 @@
"url": {
"type": "string",
"description": "URL of the vulnerability details document.",
- "format": "uri"
+ "pattern": "^(https?|ftp)://.+"
}
}
}
@@ -692,6 +737,7 @@
"$ref": "#/definitions/named_list/properties/items"
},
"tracking": {
+ "type": "object",
"description": "Describes how this vulnerability should be tracked as the project changes.",
"oneOf": [
{
@@ -730,6 +776,7 @@
"minItems": 1,
"items": {
"description": "A calculated tracking signature value and metadata.",
+ "type": "object",
"required": [
"algorithm",
"value"
@@ -804,10 +851,17 @@
"dependency": {
"type": "object",
"description": "Describes the dependency of a project where the vulnerability is located.",
+ "required": [
+ "package",
+ "version"
+ ],
"properties": {
"package": {
"type": "object",
"description": "Provides information on the package where the vulnerability is located.",
+ "required": [
+ "name"
+ ],
"properties": {
"name": {
"type": "string",
@@ -858,7 +912,6 @@
"default_branch_image": {
"type": "string",
"maxLength": 255,
- "pattern": "^[a-zA-Z0-9/_.-]+:[a-zA-Z0-9_.-]+$",
"description": "The name of the image on the default branch."
}
}
@@ -883,12 +936,16 @@
"items": {
"type": "object",
"required": [
- "cve"
+ "id"
],
"properties": {
- "cve": {
+ "id": {
"type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
+ "minLength": 1,
+ "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
+ "examples": [
+ "642735a5-1425-428d-8d4e-3c854885a3c9"
+ ]
}
}
}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.0/coverage-fuzzing-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.6/coverage-fuzzing-report-format.json
index f63ebfa2cc2..11a1375710b 100644
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.0/coverage-fuzzing-report-format.json
+++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.6/coverage-fuzzing-report-format.json
@@ -1,5 +1,6 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
+ "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/coverage-fuzzing-report-format.json",
"title": "Report format for GitLab Fuzz Testing",
"description": "This schema provides the report format for Coverage Guided Fuzz Testing (https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing).",
"definitions": {
@@ -54,6 +55,7 @@
"properties": {
"name": {
"$ref": "#/definitions/text_value",
+ "type": "string",
"minLength": 1
},
"description": {
@@ -325,9 +327,11 @@
}
},
"self": {
- "version": "14.1.0"
+ "version": "15.0.6"
},
+ "type": "object",
"required": [
+ "scan",
"version",
"vulnerabilities"
],
@@ -336,6 +340,7 @@
"scan": {
"type": "object",
"required": [
+ "analyzer",
"end_time",
"scanner",
"start_time",
@@ -346,7 +351,7 @@
"end_time": {
"type": "string",
"description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
+ "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$",
"examples": [
"2020-01-28T03:26:02"
]
@@ -384,6 +389,57 @@
}
}
},
+ "options": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "description": "A configuration option used for this scan.",
+ "required": [
+ "name",
+ "value"
+ ],
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "The configuration option name.",
+ "maxLength": 255,
+ "minLength": 1,
+ "examples": [
+ "DAST_FF_ENABLE_BAS",
+ "DOCKER_TLS_CERTDIR",
+ "DS_MAX_DEPTH",
+ "SECURE_LOG_LEVEL"
+ ]
+ },
+ "source": {
+ "type": "string",
+ "description": "The source of this option.",
+ "enum": [
+ "argument",
+ "file",
+ "env_variable",
+ "other"
+ ]
+ },
+ "value": {
+ "type": [
+ "boolean",
+ "integer",
+ "null",
+ "string"
+ ],
+ "description": "The value used for this scan.",
+ "examples": [
+ true,
+ 2,
+ null,
+ "fatal",
+ ""
+ ]
+ }
+ }
+ }
+ },
"analyzer": {
"type": "object",
"description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
@@ -412,7 +468,6 @@
},
"url": {
"type": "string",
- "format": "uri",
"pattern": "^https?://.+",
"description": "A link to more information about the analyzer.",
"examples": [
@@ -509,7 +564,7 @@
"start_time": {
"type": "string",
"description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
+ "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$",
"examples": [
"2020-02-14T16:01:59"
]
@@ -528,13 +583,47 @@
"enum": [
"coverage_fuzzing"
]
+ },
+ "primary_identifiers": {
+ "type": "array",
+ "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results",
+ "items": {
+ "type": "object",
+ "required": [
+ "type",
+ "name",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
+ "minLength": 1
+ },
+ "name": {
+ "type": "string",
+ "description": "Human-readable name of the identifier.",
+ "minLength": 1
+ },
+ "url": {
+ "type": "string",
+ "description": "URL of the identifier's documentation.",
+ "pattern": "^(https?|ftp)://.+"
+ },
+ "value": {
+ "type": "string",
+ "description": "Value of the identifier, for matching purpose.",
+ "minLength": 1
+ }
+ }
+ }
}
}
},
"schema": {
"type": "string",
"description": "URI pointing to the validating security report schema.",
- "format": "uri"
+ "pattern": "^https?://.+"
},
"version": {
"type": "string",
@@ -548,41 +637,29 @@
"type": "object",
"description": "Describes the vulnerability using GitLab Flavored Markdown",
"required": [
- "category",
- "cve",
+ "id",
"identifiers",
- "location",
- "scanner"
+ "location"
],
"properties": {
"id": {
"type": "string",
+ "minLength": 1,
"description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
"examples": [
"642735a5-1425-428d-8d4e-3c854885a3c9"
]
},
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
"name": {
"type": "string",
+ "maxLength": 255,
"description": "The name of the vulnerability. This must not include the finding's specific information."
},
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
"description": {
"type": "string",
+ "maxLength": 1048576,
"description": "A long text section describing the vulnerability more fully."
},
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
"severity": {
"type": "string",
"description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
@@ -595,43 +672,11 @@
"Critical"
]
},
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
"solution": {
"type": "string",
+ "maxLength": 7000,
"description": "Explanation of how to fix the vulnerability."
},
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
"identifiers": {
"type": "array",
"minItems": 1,
@@ -657,7 +702,7 @@
"url": {
"type": "string",
"description": "URL of the identifier's documentation.",
- "format": "uri"
+ "pattern": "^(https?|ftp)://.+"
},
"value": {
"type": "string",
@@ -683,7 +728,7 @@
"url": {
"type": "string",
"description": "URL of the vulnerability details document.",
- "format": "uri"
+ "pattern": "^(https?|ftp)://.+"
}
}
}
@@ -692,6 +737,7 @@
"$ref": "#/definitions/named_list/properties/items"
},
"tracking": {
+ "type": "object",
"description": "Describes how this vulnerability should be tracked as the project changes.",
"oneOf": [
{
@@ -730,6 +776,7 @@
"minItems": 1,
"items": {
"description": "A calculated tracking signature value and metadata.",
+ "type": "object",
"required": [
"algorithm",
"value"
@@ -847,12 +894,16 @@
"items": {
"type": "object",
"required": [
- "cve"
+ "id"
],
"properties": {
- "cve": {
+ "id": {
"type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
+ "minLength": 1,
+ "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
+ "examples": [
+ "642735a5-1425-428d-8d4e-3c854885a3c9"
+ ]
}
}
}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.6/dast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.6/dast-report-format.json
index 598f162aad2..1351cb261e0 100644
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.6/dast-report-format.json
+++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.6/dast-report-format.json
@@ -1,5 +1,6 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
+ "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/dast-report-format.json",
"title": "Report format for GitLab DAST",
"description": "This schema provides the the report format for Dynamic Application Security Testing (https://docs.gitlab.com/ee/user/application_security/dast).",
"definitions": {
@@ -54,6 +55,7 @@
"properties": {
"name": {
"$ref": "#/definitions/text_value",
+ "type": "string",
"minLength": 1
},
"description": {
@@ -325,9 +327,11 @@
}
},
"self": {
- "version": "14.0.6"
+ "version": "15.0.6"
},
+ "type": "object",
"required": [
+ "scan",
"version",
"vulnerabilities"
],
@@ -336,6 +340,7 @@
"scan": {
"type": "object",
"required": [
+ "analyzer",
"end_time",
"scanned_resources",
"scanner",
@@ -347,7 +352,7 @@
"end_time": {
"type": "string",
"description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
+ "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$",
"examples": [
"2020-01-28T03:26:02"
]
@@ -385,6 +390,57 @@
}
}
},
+ "options": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "description": "A configuration option used for this scan.",
+ "required": [
+ "name",
+ "value"
+ ],
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "The configuration option name.",
+ "maxLength": 255,
+ "minLength": 1,
+ "examples": [
+ "DAST_FF_ENABLE_BAS",
+ "DOCKER_TLS_CERTDIR",
+ "DS_MAX_DEPTH",
+ "SECURE_LOG_LEVEL"
+ ]
+ },
+ "source": {
+ "type": "string",
+ "description": "The source of this option.",
+ "enum": [
+ "argument",
+ "file",
+ "env_variable",
+ "other"
+ ]
+ },
+ "value": {
+ "type": [
+ "boolean",
+ "integer",
+ "null",
+ "string"
+ ],
+ "description": "The value used for this scan.",
+ "examples": [
+ true,
+ 2,
+ null,
+ "fatal",
+ ""
+ ]
+ }
+ }
+ }
+ },
"analyzer": {
"type": "object",
"description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
@@ -413,7 +469,6 @@
},
"url": {
"type": "string",
- "format": "uri",
"pattern": "^https?://.+",
"description": "A link to more information about the analyzer.",
"examples": [
@@ -510,7 +565,7 @@
"start_time": {
"type": "string",
"description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
+ "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$",
"examples": [
"2020-02-14T16:01:59"
]
@@ -531,6 +586,40 @@
"api_fuzzing"
]
},
+ "primary_identifiers": {
+ "type": "array",
+ "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results",
+ "items": {
+ "type": "object",
+ "required": [
+ "type",
+ "name",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
+ "minLength": 1
+ },
+ "name": {
+ "type": "string",
+ "description": "Human-readable name of the identifier.",
+ "minLength": 1
+ },
+ "url": {
+ "type": "string",
+ "description": "URL of the identifier's documentation.",
+ "pattern": "^(https?|ftp)://.+"
+ },
+ "value": {
+ "type": "string",
+ "description": "Value of the identifier, for matching purpose.",
+ "minLength": 1
+ }
+ }
+ }
+ },
"scanned_resources": {
"type": "array",
"description": "The attack surface scanned by DAST.",
@@ -576,7 +665,7 @@
"schema": {
"type": "string",
"description": "URI pointing to the validating security report schema.",
- "format": "uri"
+ "pattern": "^https?://.+"
},
"version": {
"type": "string",
@@ -590,41 +679,29 @@
"type": "object",
"description": "Describes the vulnerability using GitLab Flavored Markdown",
"required": [
- "category",
- "cve",
+ "id",
"identifiers",
- "location",
- "scanner"
+ "location"
],
"properties": {
"id": {
"type": "string",
+ "minLength": 1,
"description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
"examples": [
"642735a5-1425-428d-8d4e-3c854885a3c9"
]
},
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
"name": {
"type": "string",
+ "maxLength": 255,
"description": "The name of the vulnerability. This must not include the finding's specific information."
},
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
"description": {
"type": "string",
+ "maxLength": 1048576,
"description": "A long text section describing the vulnerability more fully."
},
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
"severity": {
"type": "string",
"description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
@@ -637,43 +714,11 @@
"Critical"
]
},
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
"solution": {
"type": "string",
+ "maxLength": 7000,
"description": "Explanation of how to fix the vulnerability."
},
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
"identifiers": {
"type": "array",
"minItems": 1,
@@ -699,7 +744,7 @@
"url": {
"type": "string",
"description": "URL of the identifier's documentation.",
- "format": "uri"
+ "pattern": "^(https?|ftp)://.+"
},
"value": {
"type": "string",
@@ -725,7 +770,7 @@
"url": {
"type": "string",
"description": "URL of the vulnerability details document.",
- "format": "uri"
+ "pattern": "^(https?|ftp)://.+"
}
}
}
@@ -734,6 +779,7 @@
"$ref": "#/definitions/named_list/properties/items"
},
"tracking": {
+ "type": "object",
"description": "Describes how this vulnerability should be tracked as the project changes.",
"oneOf": [
{
@@ -772,6 +818,7 @@
"minItems": 1,
"items": {
"description": "A calculated tracking signature value and metadata.",
+ "type": "object",
"required": [
"algorithm",
"value"
@@ -911,7 +958,6 @@
},
"value": {
"type": "string",
- "minLength": 1,
"description": "Value of the HTTP header.",
"examples": [
"*/*",
@@ -979,7 +1025,6 @@
},
"value": {
"type": "string",
- "minLength": 1,
"description": "Value of the HTTP header.",
"examples": [
"*/*",
@@ -1065,7 +1110,6 @@
},
"value": {
"type": "string",
- "minLength": 1,
"description": "Value of the HTTP header.",
"examples": [
"*/*",
@@ -1133,7 +1177,6 @@
},
"value": {
"type": "string",
- "minLength": 1,
"description": "Value of the HTTP header.",
"examples": [
"*/*",
@@ -1235,14 +1278,6 @@
}
}
}
- },
- "discovered_at": {
- "type": "string",
- "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss.sss, representing when the vulnerability was discovered",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}\\.\\d{3}$",
- "examples": [
- "2020-01-28T03:26:02.956"
- ]
}
}
}
@@ -1264,12 +1299,16 @@
"items": {
"type": "object",
"required": [
- "cve"
+ "id"
],
"properties": {
- "cve": {
+ "id": {
"type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
+ "minLength": 1,
+ "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
+ "examples": [
+ "642735a5-1425-428d-8d4e-3c854885a3c9"
+ ]
}
}
}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.0/dependency-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.6/dependency-scanning-report-format.json
index 6f2c3740b09..e4b02362cb1 100644
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.0/dependency-scanning-report-format.json
+++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.6/dependency-scanning-report-format.json
@@ -1,5 +1,6 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
+ "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/dependency-scanning-report-format.json",
"title": "Report format for GitLab Dependency Scanning",
"description": "This schema provides the the report format for Dependency Scanning analyzers (https://docs.gitlab.com/ee/user/application_security/dependency_scanning).",
"definitions": {
@@ -54,6 +55,7 @@
"properties": {
"name": {
"$ref": "#/definitions/text_value",
+ "type": "string",
"minLength": 1
},
"description": {
@@ -325,10 +327,12 @@
}
},
"self": {
- "version": "14.1.0"
+ "version": "15.0.6"
},
+ "type": "object",
"required": [
"dependency_files",
+ "scan",
"version",
"vulnerabilities"
],
@@ -337,6 +341,7 @@
"scan": {
"type": "object",
"required": [
+ "analyzer",
"end_time",
"scanner",
"start_time",
@@ -347,7 +352,7 @@
"end_time": {
"type": "string",
"description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
+ "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$",
"examples": [
"2020-01-28T03:26:02"
]
@@ -385,6 +390,57 @@
}
}
},
+ "options": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "description": "A configuration option used for this scan.",
+ "required": [
+ "name",
+ "value"
+ ],
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "The configuration option name.",
+ "maxLength": 255,
+ "minLength": 1,
+ "examples": [
+ "DAST_FF_ENABLE_BAS",
+ "DOCKER_TLS_CERTDIR",
+ "DS_MAX_DEPTH",
+ "SECURE_LOG_LEVEL"
+ ]
+ },
+ "source": {
+ "type": "string",
+ "description": "The source of this option.",
+ "enum": [
+ "argument",
+ "file",
+ "env_variable",
+ "other"
+ ]
+ },
+ "value": {
+ "type": [
+ "boolean",
+ "integer",
+ "null",
+ "string"
+ ],
+ "description": "The value used for this scan.",
+ "examples": [
+ true,
+ 2,
+ null,
+ "fatal",
+ ""
+ ]
+ }
+ }
+ }
+ },
"analyzer": {
"type": "object",
"description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
@@ -413,7 +469,6 @@
},
"url": {
"type": "string",
- "format": "uri",
"pattern": "^https?://.+",
"description": "A link to more information about the analyzer.",
"examples": [
@@ -510,7 +565,7 @@
"start_time": {
"type": "string",
"description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
+ "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$",
"examples": [
"2020-02-14T16:01:59"
]
@@ -529,13 +584,47 @@
"enum": [
"dependency_scanning"
]
+ },
+ "primary_identifiers": {
+ "type": "array",
+ "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results",
+ "items": {
+ "type": "object",
+ "required": [
+ "type",
+ "name",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
+ "minLength": 1
+ },
+ "name": {
+ "type": "string",
+ "description": "Human-readable name of the identifier.",
+ "minLength": 1
+ },
+ "url": {
+ "type": "string",
+ "description": "URL of the identifier's documentation.",
+ "pattern": "^(https?|ftp)://.+"
+ },
+ "value": {
+ "type": "string",
+ "description": "Value of the identifier, for matching purpose.",
+ "minLength": 1
+ }
+ }
+ }
}
}
},
"schema": {
"type": "string",
"description": "URI pointing to the validating security report schema.",
- "format": "uri"
+ "pattern": "^https?://.+"
},
"version": {
"type": "string",
@@ -549,41 +638,29 @@
"type": "object",
"description": "Describes the vulnerability using GitLab Flavored Markdown",
"required": [
- "category",
- "cve",
+ "id",
"identifiers",
- "location",
- "scanner"
+ "location"
],
"properties": {
"id": {
"type": "string",
+ "minLength": 1,
"description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
"examples": [
"642735a5-1425-428d-8d4e-3c854885a3c9"
]
},
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
"name": {
"type": "string",
+ "maxLength": 255,
"description": "The name of the vulnerability. This must not include the finding's specific information."
},
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
"description": {
"type": "string",
+ "maxLength": 1048576,
"description": "A long text section describing the vulnerability more fully."
},
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
"severity": {
"type": "string",
"description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
@@ -596,43 +673,11 @@
"Critical"
]
},
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
"solution": {
"type": "string",
+ "maxLength": 7000,
"description": "Explanation of how to fix the vulnerability."
},
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
"identifiers": {
"type": "array",
"minItems": 1,
@@ -658,7 +703,7 @@
"url": {
"type": "string",
"description": "URL of the identifier's documentation.",
- "format": "uri"
+ "pattern": "^(https?|ftp)://.+"
},
"value": {
"type": "string",
@@ -684,7 +729,7 @@
"url": {
"type": "string",
"description": "URL of the vulnerability details document.",
- "format": "uri"
+ "pattern": "^(https?|ftp)://.+"
}
}
}
@@ -693,6 +738,7 @@
"$ref": "#/definitions/named_list/properties/items"
},
"tracking": {
+ "type": "object",
"description": "Describes how this vulnerability should be tracked as the project changes.",
"oneOf": [
{
@@ -731,6 +777,7 @@
"minItems": 1,
"items": {
"description": "A calculated tracking signature value and metadata.",
+ "type": "object",
"required": [
"algorithm",
"value"
@@ -809,10 +856,17 @@
"dependency": {
"type": "object",
"description": "Describes the dependency of a project where the vulnerability is located.",
+ "required": [
+ "package",
+ "version"
+ ],
"properties": {
"package": {
"type": "object",
"description": "Provides information on the package where the vulnerability is located.",
+ "required": [
+ "name"
+ ],
"properties": {
"name": {
"type": "string",
@@ -872,12 +926,16 @@
"items": {
"type": "object",
"required": [
- "cve"
+ "id"
],
"properties": {
- "cve": {
+ "id": {
"type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
+ "minLength": 1,
+ "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
+ "examples": [
+ "642735a5-1425-428d-8d4e-3c854885a3c9"
+ ]
}
}
}
@@ -919,10 +977,17 @@
"items": {
"type": "object",
"description": "Describes the dependency of a project where the vulnerability is located.",
+ "required": [
+ "package",
+ "version"
+ ],
"properties": {
"package": {
"type": "object",
"description": "Provides information on the package where the vulnerability is located.",
+ "required": [
+ "name"
+ ],
"properties": {
"name": {
"type": "string",
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.0/sast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.6/sast-report-format.json
index 5c7f636e169..e4cb5fb2985 100644
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.0/sast-report-format.json
+++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.6/sast-report-format.json
@@ -1,5 +1,6 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
+ "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/sast-report-format.json",
"title": "Report format for GitLab SAST",
"description": "This schema provides the report format for Static Application Security Testing analyzers (https://docs.gitlab.com/ee/user/application_security/sast).",
"definitions": {
@@ -54,6 +55,7 @@
"properties": {
"name": {
"$ref": "#/definitions/text_value",
+ "type": "string",
"minLength": 1
},
"description": {
@@ -325,9 +327,11 @@
}
},
"self": {
- "version": "14.1.0"
+ "version": "15.0.6"
},
+ "type": "object",
"required": [
+ "scan",
"version",
"vulnerabilities"
],
@@ -336,6 +340,7 @@
"scan": {
"type": "object",
"required": [
+ "analyzer",
"end_time",
"scanner",
"start_time",
@@ -346,7 +351,7 @@
"end_time": {
"type": "string",
"description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
+ "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$",
"examples": [
"2020-01-28T03:26:02"
]
@@ -384,6 +389,57 @@
}
}
},
+ "options": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "description": "A configuration option used for this scan.",
+ "required": [
+ "name",
+ "value"
+ ],
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "The configuration option name.",
+ "maxLength": 255,
+ "minLength": 1,
+ "examples": [
+ "DAST_FF_ENABLE_BAS",
+ "DOCKER_TLS_CERTDIR",
+ "DS_MAX_DEPTH",
+ "SECURE_LOG_LEVEL"
+ ]
+ },
+ "source": {
+ "type": "string",
+ "description": "The source of this option.",
+ "enum": [
+ "argument",
+ "file",
+ "env_variable",
+ "other"
+ ]
+ },
+ "value": {
+ "type": [
+ "boolean",
+ "integer",
+ "null",
+ "string"
+ ],
+ "description": "The value used for this scan.",
+ "examples": [
+ true,
+ 2,
+ null,
+ "fatal",
+ ""
+ ]
+ }
+ }
+ }
+ },
"analyzer": {
"type": "object",
"description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
@@ -412,7 +468,6 @@
},
"url": {
"type": "string",
- "format": "uri",
"pattern": "^https?://.+",
"description": "A link to more information about the analyzer.",
"examples": [
@@ -509,7 +564,7 @@
"start_time": {
"type": "string",
"description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
+ "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$",
"examples": [
"2020-02-14T16:01:59"
]
@@ -528,13 +583,47 @@
"enum": [
"sast"
]
+ },
+ "primary_identifiers": {
+ "type": "array",
+ "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results",
+ "items": {
+ "type": "object",
+ "required": [
+ "type",
+ "name",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
+ "minLength": 1
+ },
+ "name": {
+ "type": "string",
+ "description": "Human-readable name of the identifier.",
+ "minLength": 1
+ },
+ "url": {
+ "type": "string",
+ "description": "URL of the identifier's documentation.",
+ "pattern": "^(https?|ftp)://.+"
+ },
+ "value": {
+ "type": "string",
+ "description": "Value of the identifier, for matching purpose.",
+ "minLength": 1
+ }
+ }
+ }
}
}
},
"schema": {
"type": "string",
"description": "URI pointing to the validating security report schema.",
- "format": "uri"
+ "pattern": "^https?://.+"
},
"version": {
"type": "string",
@@ -548,41 +637,29 @@
"type": "object",
"description": "Describes the vulnerability using GitLab Flavored Markdown",
"required": [
- "category",
- "cve",
+ "id",
"identifiers",
- "location",
- "scanner"
+ "location"
],
"properties": {
"id": {
"type": "string",
+ "minLength": 1,
"description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
"examples": [
"642735a5-1425-428d-8d4e-3c854885a3c9"
]
},
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
"name": {
"type": "string",
+ "maxLength": 255,
"description": "The name of the vulnerability. This must not include the finding's specific information."
},
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
"description": {
"type": "string",
+ "maxLength": 1048576,
"description": "A long text section describing the vulnerability more fully."
},
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
"severity": {
"type": "string",
"description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
@@ -595,43 +672,11 @@
"Critical"
]
},
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
"solution": {
"type": "string",
+ "maxLength": 7000,
"description": "Explanation of how to fix the vulnerability."
},
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
"identifiers": {
"type": "array",
"minItems": 1,
@@ -657,7 +702,7 @@
"url": {
"type": "string",
"description": "URL of the identifier's documentation.",
- "format": "uri"
+ "pattern": "^(https?|ftp)://.+"
},
"value": {
"type": "string",
@@ -683,7 +728,7 @@
"url": {
"type": "string",
"description": "URL of the vulnerability details document.",
- "format": "uri"
+ "pattern": "^(https?|ftp)://.+"
}
}
}
@@ -692,6 +737,7 @@
"$ref": "#/definitions/named_list/properties/items"
},
"tracking": {
+ "type": "object",
"description": "Describes how this vulnerability should be tracked as the project changes.",
"oneOf": [
{
@@ -730,6 +776,7 @@
"minItems": 1,
"items": {
"description": "A calculated tracking signature value and metadata.",
+ "type": "object",
"required": [
"algorithm",
"value"
@@ -842,12 +889,16 @@
"items": {
"type": "object",
"required": [
- "cve"
+ "id"
],
"properties": {
- "cve": {
+ "id": {
"type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
+ "minLength": 1,
+ "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
+ "examples": [
+ "642735a5-1425-428d-8d4e-3c854885a3c9"
+ ]
}
}
}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.0/secret-detection-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.6/secret-detection-report-format.json
index a87388c45e7..5eb52b11efe 100644
--- a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.0/secret-detection-report-format.json
+++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.6/secret-detection-report-format.json
@@ -1,5 +1,6 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
+ "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/secret-detection-report-format.json",
"title": "Report format for GitLab Secret Detection",
"description": "This schema provides the the report format for the Secret Detection analyzer (https://docs.gitlab.com/ee/user/application_security/secret_detection)",
"definitions": {
@@ -54,6 +55,7 @@
"properties": {
"name": {
"$ref": "#/definitions/text_value",
+ "type": "string",
"minLength": 1
},
"description": {
@@ -325,9 +327,11 @@
}
},
"self": {
- "version": "14.1.0"
+ "version": "15.0.6"
},
+ "type": "object",
"required": [
+ "scan",
"version",
"vulnerabilities"
],
@@ -336,6 +340,7 @@
"scan": {
"type": "object",
"required": [
+ "analyzer",
"end_time",
"scanner",
"start_time",
@@ -346,7 +351,7 @@
"end_time": {
"type": "string",
"description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
+ "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$",
"examples": [
"2020-01-28T03:26:02"
]
@@ -384,6 +389,57 @@
}
}
},
+ "options": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "description": "A configuration option used for this scan.",
+ "required": [
+ "name",
+ "value"
+ ],
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "The configuration option name.",
+ "maxLength": 255,
+ "minLength": 1,
+ "examples": [
+ "DAST_FF_ENABLE_BAS",
+ "DOCKER_TLS_CERTDIR",
+ "DS_MAX_DEPTH",
+ "SECURE_LOG_LEVEL"
+ ]
+ },
+ "source": {
+ "type": "string",
+ "description": "The source of this option.",
+ "enum": [
+ "argument",
+ "file",
+ "env_variable",
+ "other"
+ ]
+ },
+ "value": {
+ "type": [
+ "boolean",
+ "integer",
+ "null",
+ "string"
+ ],
+ "description": "The value used for this scan.",
+ "examples": [
+ true,
+ 2,
+ null,
+ "fatal",
+ ""
+ ]
+ }
+ }
+ }
+ },
"analyzer": {
"type": "object",
"description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
@@ -412,7 +468,6 @@
},
"url": {
"type": "string",
- "format": "uri",
"pattern": "^https?://.+",
"description": "A link to more information about the analyzer.",
"examples": [
@@ -509,7 +564,7 @@
"start_time": {
"type": "string",
"description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
+ "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$",
"examples": [
"2020-02-14T16:01:59"
]
@@ -528,13 +583,47 @@
"enum": [
"secret_detection"
]
+ },
+ "primary_identifiers": {
+ "type": "array",
+ "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results",
+ "items": {
+ "type": "object",
+ "required": [
+ "type",
+ "name",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
+ "minLength": 1
+ },
+ "name": {
+ "type": "string",
+ "description": "Human-readable name of the identifier.",
+ "minLength": 1
+ },
+ "url": {
+ "type": "string",
+ "description": "URL of the identifier's documentation.",
+ "pattern": "^(https?|ftp)://.+"
+ },
+ "value": {
+ "type": "string",
+ "description": "Value of the identifier, for matching purpose.",
+ "minLength": 1
+ }
+ }
+ }
}
}
},
"schema": {
"type": "string",
"description": "URI pointing to the validating security report schema.",
- "format": "uri"
+ "pattern": "^https?://.+"
},
"version": {
"type": "string",
@@ -548,41 +637,29 @@
"type": "object",
"description": "Describes the vulnerability using GitLab Flavored Markdown",
"required": [
- "category",
- "cve",
+ "id",
"identifiers",
- "location",
- "scanner"
+ "location"
],
"properties": {
"id": {
"type": "string",
+ "minLength": 1,
"description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
"examples": [
"642735a5-1425-428d-8d4e-3c854885a3c9"
]
},
- "category": {
- "type": "string",
- "minLength": 1,
- "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
- },
"name": {
"type": "string",
+ "maxLength": 255,
"description": "The name of the vulnerability. This must not include the finding's specific information."
},
- "message": {
- "type": "string",
- "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
- },
"description": {
"type": "string",
+ "maxLength": 1048576,
"description": "A long text section describing the vulnerability more fully."
},
- "cve": {
- "type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
- },
"severity": {
"type": "string",
"description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
@@ -595,43 +672,11 @@
"Critical"
]
},
- "confidence": {
- "type": "string",
- "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
- "enum": [
- "Ignore",
- "Unknown",
- "Experimental",
- "Low",
- "Medium",
- "High",
- "Confirmed"
- ]
- },
"solution": {
"type": "string",
+ "maxLength": 7000,
"description": "Explanation of how to fix the vulnerability."
},
- "scanner": {
- "description": "Describes the scanner used to find this vulnerability.",
- "type": "object",
- "required": [
- "id",
- "name"
- ],
- "properties": {
- "id": {
- "type": "string",
- "minLength": 1,
- "description": "The scanner's ID, as a snake_case string."
- },
- "name": {
- "type": "string",
- "minLength": 1,
- "description": "Human-readable name of the scanner."
- }
- }
- },
"identifiers": {
"type": "array",
"minItems": 1,
@@ -657,7 +702,7 @@
"url": {
"type": "string",
"description": "URL of the identifier's documentation.",
- "format": "uri"
+ "pattern": "^(https?|ftp)://.+"
},
"value": {
"type": "string",
@@ -683,7 +728,7 @@
"url": {
"type": "string",
"description": "URL of the vulnerability details document.",
- "format": "uri"
+ "pattern": "^(https?|ftp)://.+"
}
}
}
@@ -692,6 +737,7 @@
"$ref": "#/definitions/named_list/properties/items"
},
"tracking": {
+ "type": "object",
"description": "Describes how this vulnerability should be tracked as the project changes.",
"oneOf": [
{
@@ -730,6 +776,7 @@
"minItems": 1,
"items": {
"description": "A calculated tracking signature value and metadata.",
+ "type": "object",
"required": [
"algorithm",
"value"
@@ -796,6 +843,7 @@
"required": [
"commit"
],
+ "type": "object",
"properties": {
"file": {
"type": "string",
@@ -865,12 +913,16 @@
"items": {
"type": "object",
"required": [
- "cve"
+ "id"
],
"properties": {
- "cve": {
+ "id": {
"type": "string",
- "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
+ "minLength": 1,
+ "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
+ "examples": [
+ "642735a5-1425-428d-8d4e-3c854885a3c9"
+ ]
}
}
}
diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb
index d2dc712e366..4bc2f6c7be7 100644
--- a/lib/gitlab/ci/pipeline/chain/command.rb
+++ b/lib/gitlab/ci/pipeline/chain/command.rb
@@ -13,7 +13,8 @@ module Gitlab
:seeds_block, :variables_attributes, :push_options,
:chat_data, :allow_mirror_update, :bridge, :content, :dry_run, :logger,
# These attributes are set by Chains during processing:
- :config_content, :yaml_processor_result, :workflow_rules_result, :pipeline_seed
+ :config_content, :yaml_processor_result, :workflow_rules_result, :pipeline_seed,
+ :pipeline_config
) do
include Gitlab::Utils::StrongMemoize
diff --git a/lib/gitlab/ci/pipeline/chain/config/content.rb b/lib/gitlab/ci/pipeline/chain/config/content.rb
index d41213ef6dd..779aac7d520 100644
--- a/lib/gitlab/ci/pipeline/chain/config/content.rb
+++ b/lib/gitlab/ci/pipeline/chain/config/content.rb
@@ -14,6 +14,7 @@ module Gitlab
@pipeline.build_pipeline_config(content: pipeline_config.content)
@command.config_content = pipeline_config.content
@pipeline.config_source = pipeline_config.source
+ @command.pipeline_config = pipeline_config
else
error('Missing CI config file')
end
diff --git a/lib/gitlab/ci/pipeline/chain/config/process.rb b/lib/gitlab/ci/pipeline/chain/config/process.rb
index ad6b2fd3411..4976e075727 100644
--- a/lib/gitlab/ci/pipeline/chain/config/process.rb
+++ b/lib/gitlab/ci/pipeline/chain/config/process.rb
@@ -20,6 +20,7 @@ module Gitlab
source: @pipeline.source,
user: current_user,
parent_pipeline: parent_pipeline,
+ pipeline_config: @command.pipeline_config,
logger: logger
}
)
diff --git a/lib/gitlab/ci/pipeline/chain/limit/activity.rb b/lib/gitlab/ci/pipeline/chain/limit/activity.rb
deleted file mode 100644
index ef9235477db..00000000000
--- a/lib/gitlab/ci/pipeline/chain/limit/activity.rb
+++ /dev/null
@@ -1,23 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Ci
- module Pipeline
- module Chain
- module Limit
- class Activity < Chain::Base
- def perform!
- # to be overridden in EE
- end
-
- def break?
- false # to be overridden in EE
- end
- end
- end
- end
- end
- end
-end
-
-Gitlab::Ci::Pipeline::Chain::Limit::Activity.prepend_mod_with('Gitlab::Ci::Pipeline::Chain::Limit::Activity')
diff --git a/lib/gitlab/ci/pipeline/duration.rb b/lib/gitlab/ci/pipeline/duration.rb
index e8a991026b5..573d4c25b91 100644
--- a/lib/gitlab/ci/pipeline/duration.rb
+++ b/lib/gitlab/ci/pipeline/duration.rb
@@ -82,6 +82,8 @@ module Gitlab
module Duration
extend self
+ STATUSES = %w[success failed running canceled].freeze
+
Period = Struct.new(:first, :last) do
def duration
last - first
@@ -90,14 +92,15 @@ module Gitlab
# rubocop: disable CodeReuse/ActiveRecord
def from_pipeline(pipeline)
- status = %w[success failed running canceled]
- builds = pipeline.processables.latest
- .where(status: status).where.not(started_at: nil).order(:started_at)
+ builds =
+ self_and_downstreams_builds_of_pipeline(pipeline)
from_builds(builds)
end
# rubocop: enable CodeReuse/ActiveRecord
+ private
+
def from_builds(builds)
now = Time.now
@@ -113,8 +116,6 @@ module Gitlab
process_duration(process_periods(periods))
end
- private
-
def process_periods(periods)
return periods if periods.empty?
@@ -139,6 +140,20 @@ module Gitlab
end
# rubocop: disable CodeReuse/ActiveRecord
+ def self_and_downstreams_builds_of_pipeline(pipeline)
+ ::Ci::Build
+ .select(:id, :type, :started_at, :finished_at)
+ .in_pipelines(
+ pipeline.self_and_downstreams.select(:id)
+ )
+ .with_status(STATUSES)
+ .latest
+ .where.not(started_at: nil)
+ .order(:started_at)
+ end
+ # rubocop: enable CodeReuse/ActiveRecord
+
+ # rubocop: disable CodeReuse/ActiveRecord
def process_duration(periods)
periods.sum(&:duration)
end
diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb
index 484e18c6979..98f488d0f38 100644
--- a/lib/gitlab/ci/pipeline/seed/build.rb
+++ b/lib/gitlab/ci/pipeline/seed/build.rb
@@ -123,6 +123,7 @@ module Gitlab
end
@needs_attributes.flat_map do |need|
+ # We ignore the optional needed job in case it is excluded from the pipeline due to the job's rules.
next if need[:optional]
result = need_present?(need)
diff --git a/lib/gitlab/ci/pipeline/seed/build/cache.rb b/lib/gitlab/ci/pipeline/seed/build/cache.rb
index 409b6658cc0..936344b9ae8 100644
--- a/lib/gitlab/ci/pipeline/seed/build/cache.rb
+++ b/lib/gitlab/ci/pipeline/seed/build/cache.rb
@@ -16,6 +16,7 @@ module Gitlab
@when = local_cache.delete(:when)
@unprotect = local_cache.delete(:unprotect)
@custom_key_prefix = custom_key_prefix
+ @fallback_keys = local_cache.delete(:fallback_keys)
raise ArgumentError, "unknown cache keys: #{local_cache.keys}" if local_cache.any?
end
@@ -27,7 +28,8 @@ module Gitlab
policy: @policy,
untracked: @untracked,
when: @when,
- unprotect: @unprotect
+ unprotect: @unprotect,
+ fallback_keys: @fallback_keys
}.compact
end
diff --git a/lib/gitlab/ci/project_config.rb b/lib/gitlab/ci/project_config.rb
index ded6877ef29..00b2ad58428 100644
--- a/lib/gitlab/ci/project_config.rb
+++ b/lib/gitlab/ci/project_config.rb
@@ -26,6 +26,7 @@ module Gitlab
end
delegate :content, :source, to: :@config, allow_nil: true
+ delegate :internal_include_prepended?, to: :@config
def exists?
!!@config&.exists?
diff --git a/lib/gitlab/ci/project_config/auto_devops.rb b/lib/gitlab/ci/project_config/auto_devops.rb
index c6905f480a2..c5f010ebaea 100644
--- a/lib/gitlab/ci/project_config/auto_devops.rb
+++ b/lib/gitlab/ci/project_config/auto_devops.rb
@@ -13,6 +13,10 @@ module Gitlab
end
end
+ def internal_include_prepended?
+ true
+ end
+
def source
:auto_devops_source
end
diff --git a/lib/gitlab/ci/project_config/external_project.rb b/lib/gitlab/ci/project_config/external_project.rb
index 0ed5d6fa226..0afdab23886 100644
--- a/lib/gitlab/ci/project_config/external_project.rb
+++ b/lib/gitlab/ci/project_config/external_project.rb
@@ -17,6 +17,10 @@ module Gitlab
end
end
+ def internal_include_prepended?
+ true
+ end
+
def source
:external_project_source
end
diff --git a/lib/gitlab/ci/project_config/remote.rb b/lib/gitlab/ci/project_config/remote.rb
index cf1292706d2..19cbf8e9c1e 100644
--- a/lib/gitlab/ci/project_config/remote.rb
+++ b/lib/gitlab/ci/project_config/remote.rb
@@ -12,6 +12,10 @@ module Gitlab
end
end
+ def internal_include_prepended?
+ true
+ end
+
def source
:remote_source
end
diff --git a/lib/gitlab/ci/project_config/repository.rb b/lib/gitlab/ci/project_config/repository.rb
index 435ad4d42fe..272425fd546 100644
--- a/lib/gitlab/ci/project_config/repository.rb
+++ b/lib/gitlab/ci/project_config/repository.rb
@@ -12,6 +12,10 @@ module Gitlab
end
end
+ def internal_include_prepended?
+ true
+ end
+
def source
:repository_source
end
diff --git a/lib/gitlab/ci/project_config/source.rb b/lib/gitlab/ci/project_config/source.rb
index ebe5728163b..9a4a6394fa1 100644
--- a/lib/gitlab/ci/project_config/source.rb
+++ b/lib/gitlab/ci/project_config/source.rb
@@ -24,6 +24,11 @@ module Gitlab
raise NotImplementedError
end
+ # Indicates if we are prepending the content with an "internal" `include`
+ def internal_include_prepended?
+ false
+ end
+
def source
raise NotImplementedError
end
diff --git a/lib/gitlab/ci/reports/security/finding.rb b/lib/gitlab/ci/reports/security/finding.rb
index 92a91854358..bf48c7d0bb7 100644
--- a/lib/gitlab/ci/reports/security/finding.rb
+++ b/lib/gitlab/ci/reports/security/finding.rb
@@ -29,12 +29,13 @@ module Gitlab
attr_reader :signatures
attr_reader :project_id
attr_reader :original_data
+ attr_reader :found_by_pipeline
delegate :file_path, :start_line, :end_line, to: :location
alias_method :cve, :compare_key
- def initialize(compare_key:, identifiers:, flags: [], links: [], remediations: [], location:, evidence:, metadata_version:, name:, original_data:, report_type:, scanner:, scan:, uuid:, confidence: nil, severity: nil, details: {}, signatures: [], project_id: nil, vulnerability_finding_signatures_enabled: false) # rubocop:disable Metrics/ParameterLists
+ def initialize(compare_key:, identifiers:, flags: [], links: [], remediations: [], location:, evidence:, metadata_version:, name:, original_data:, report_type:, scanner:, scan:, uuid:, confidence: nil, severity: nil, details: {}, signatures: [], project_id: nil, vulnerability_finding_signatures_enabled: false, found_by_pipeline: nil) # rubocop:disable Metrics/ParameterLists
@compare_key = compare_key
@confidence = confidence
@identifiers = identifiers
@@ -55,6 +56,7 @@ module Gitlab
@signatures = signatures
@project_id = project_id
@vulnerability_finding_signatures_enabled = vulnerability_finding_signatures_enabled
+ @found_by_pipeline = found_by_pipeline
@project_fingerprint = generate_project_fingerprint
end
@@ -188,6 +190,10 @@ module Gitlab
original_data['assets'] || []
end
+ def raw_source_code_extract
+ original_data['raw_source_code_extract']
+ end
+
# Returns either the max priority signature hex
# or the location fingerprint
def location_fingerprint
diff --git a/lib/gitlab/ci/reports/security/report.rb b/lib/gitlab/ci/reports/security/report.rb
index 54b21da5436..2287c397c2b 100644
--- a/lib/gitlab/ci/reports/security/report.rb
+++ b/lib/gitlab/ci/reports/security/report.rb
@@ -5,8 +5,9 @@ module Gitlab
module Reports
module Security
class Report
- attr_reader :created_at, :type, :pipeline, :findings, :scanners, :identifiers
- attr_accessor :scan, :scanned_resources, :errors, :analyzer, :version, :schema_validation_status, :warnings
+ attr_reader :created_at, :type, :findings, :scanners, :identifiers
+ attr_accessor :scan, :pipeline, :scanned_resources, :errors,
+ :analyzer, :version, :schema_validation_status, :warnings
delegate :project_id, to: :pipeline
delegate :project, to: :pipeline
diff --git a/lib/gitlab/ci/reports/security/vulnerability_reports_comparer.rb b/lib/gitlab/ci/reports/security/vulnerability_reports_comparer.rb
deleted file mode 100644
index 4be4cf62e7b..00000000000
--- a/lib/gitlab/ci/reports/security/vulnerability_reports_comparer.rb
+++ /dev/null
@@ -1,165 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Ci
- module Reports
- module Security
- class VulnerabilityReportsComparer
- include Gitlab::Utils::StrongMemoize
-
- attr_reader :base_report, :head_report
-
- ACCEPTABLE_REPORT_AGE = 1.week
-
- def initialize(project, base_report, head_report)
- @base_report = base_report
- @head_report = head_report
-
- @signatures_enabled = project.licensed_feature_available?(:vulnerability_finding_signatures)
-
- if @signatures_enabled
- @added_findings = []
- @fixed_findings = []
- calculate_changes
- end
- end
-
- def base_report_created_at
- @base_report.created_at
- end
-
- def head_report_created_at
- @head_report.created_at
- end
-
- def base_report_out_of_date
- return false unless @base_report.created_at
-
- ACCEPTABLE_REPORT_AGE.ago > @base_report.created_at
- end
-
- def added
- strong_memoize(:added) do
- if @signatures_enabled
- @added_findings
- else
- head_report.findings - base_report.findings
- end
- end
- end
-
- def fixed
- strong_memoize(:fixed) do
- if @signatures_enabled
- @fixed_findings
- else
- base_report.findings - head_report.findings
- end
- end
- end
-
- private
-
- def calculate_changes
- # This is a deconstructed version of the eql? method on
- # Ci::Reports::Security::Finding. It:
- #
- # * precomputes for the head_findings (using FindingMatcher):
- # * sets of signature shas grouped by priority
- # * mappings of signature shas to the head finding object
- #
- # These are then used when iterating the base findings to perform
- # fast(er) prioritized, signature-based comparisons between each base finding
- # and the head findings.
- #
- # Both the head_findings and base_findings arrays are iterated once
-
- base_findings = base_report.findings
- head_findings = head_report.findings
-
- matcher = FindingMatcher.new(head_findings)
-
- base_findings.each do |base_finding|
- next if base_finding.requires_manual_resolution?
-
- matched_head_finding = matcher.find_and_remove_match!(base_finding)
-
- @fixed_findings << base_finding if matched_head_finding.nil?
- end
-
- @added_findings = matcher.unmatched_head_findings.values
- end
- end
-
- class FindingMatcher
- attr_reader :unmatched_head_findings, :head_findings
-
- include Gitlab::Utils::StrongMemoize
-
- def initialize(head_findings)
- @head_findings = head_findings
- @unmatched_head_findings = @head_findings.index_by(&:object_id)
- end
-
- def find_and_remove_match!(base_finding)
- matched_head_finding = find_matched_head_finding_for(base_finding)
-
- # no signatures matched, so check the normal uuids of the base and head findings
- # for a match
- matched_head_finding = head_signatures_shas[base_finding.uuid] if matched_head_finding.nil?
-
- @unmatched_head_findings.delete(matched_head_finding.object_id) unless matched_head_finding.nil?
-
- matched_head_finding
- end
-
- private
-
- def find_matched_head_finding_for(base_finding)
- base_signature = sorted_signatures_for(base_finding).find do |signature|
- # at this point a head_finding exists that has a signature with a
- # matching priority, and a matching sha --> lookup the actual finding
- # object from head_signatures_shas
- head_signatures_shas[signature.signature_sha].eql?(base_finding)
- end
-
- base_signature.present? ? head_signatures_shas[base_signature.signature_sha] : nil
- end
-
- def sorted_signatures_for(base_finding)
- base_finding.signatures.select { |signature| head_finding_signature?(signature) }
- .sort_by { |sig| -sig.priority }
- end
-
- def head_finding_signature?(signature)
- head_signatures_priorities[signature.priority].include?(signature.signature_sha)
- end
-
- def head_signatures_priorities
- strong_memoize(:head_signatures_priorities) do
- signatures_priorities = Hash.new { |hash, key| hash[key] = Set.new }
-
- head_findings.each_with_object(signatures_priorities) do |head_finding, memo|
- head_finding.signatures.each do |signature|
- memo[signature.priority].add(signature.signature_sha)
- end
- end
- end
- end
-
- def head_signatures_shas
- strong_memoize(:head_signatures_shas) do
- head_findings.each_with_object({}) do |head_finding, memo|
- head_finding.signatures.each do |signature|
- memo[signature.signature_sha] = head_finding
- end
- # for the final uuid check when no signatures have matched
- memo[head_finding.uuid] = head_finding
- end
- end
- end
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/ci/resource_groups/logger.rb b/lib/gitlab/ci/resource_groups/logger.rb
new file mode 100644
index 00000000000..9c93ee95bc7
--- /dev/null
+++ b/lib/gitlab/ci/resource_groups/logger.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ module ResourceGroups
+ class Logger < ::Gitlab::JsonLogger
+ def self.file_name_noext
+ 'ci_resource_groups_json'
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/runner_releases.rb b/lib/gitlab/ci/runner_releases.rb
index dab24bfd501..a27dd3896e1 100644
--- a/lib/gitlab/ci/runner_releases.rb
+++ b/lib/gitlab/ci/runner_releases.rb
@@ -15,9 +15,14 @@ module Gitlab
reset_backoff!
end
+ def enabled?
+ ::Gitlab::CurrentSettings.current_application_settings.update_runner_versions_enabled?
+ end
+
# Returns a sorted list of the publicly available GitLab Runner releases
#
def releases
+ return unless enabled?
return if backoff_active?
Rails.cache.fetch(
diff --git a/lib/gitlab/ci/secure_files/cer.rb b/lib/gitlab/ci/secure_files/cer.rb
index 45d2898c29b..3340afa8f2a 100644
--- a/lib/gitlab/ci/secure_files/cer.rb
+++ b/lib/gitlab/ci/secure_files/cer.rb
@@ -36,7 +36,7 @@ module Gitlab
private
def expires_at
- certificate_data.not_before
+ certificate_data.not_after
end
def id
diff --git a/lib/gitlab/ci/secure_files/p12.rb b/lib/gitlab/ci/secure_files/p12.rb
index 1006a4d05b2..04cd4243bb0 100644
--- a/lib/gitlab/ci/secure_files/p12.rb
+++ b/lib/gitlab/ci/secure_files/p12.rb
@@ -36,7 +36,7 @@ module Gitlab
private
def expires_at
- certificate_data.not_before
+ certificate_data.not_after
end
def serial
diff --git a/lib/gitlab/ci/status/build/erased.rb b/lib/gitlab/ci/status/build/erased.rb
index d74cfc1ee77..c3430b8cc1c 100644
--- a/lib/gitlab/ci/status/build/erased.rb
+++ b/lib/gitlab/ci/status/build/erased.rb
@@ -7,8 +7,8 @@ module Gitlab
class Erased < Status::Extended
def illustration
{
- image: 'illustrations/erased-log_empty.svg',
- size: 'svg-430',
+ image: 'illustrations/empty-state/empty-projects-deleted-md.svg',
+ size: 'svg-150',
title: _('Job has been erased')
}
end
diff --git a/lib/gitlab/ci/status/build/factory.rb b/lib/gitlab/ci/status/build/factory.rb
index a4434e2c144..54f6784b847 100644
--- a/lib/gitlab/ci/status/build/factory.rb
+++ b/lib/gitlab/ci/status/build/factory.rb
@@ -11,12 +11,12 @@ module Gitlab
Status::Build::Manual,
Status::Build::Canceled,
Status::Build::Created,
- Status::Build::WaitingForResource,
Status::Build::Preparing,
Status::Build::Pending,
Status::Build::Skipped,
Status::Build::WaitingForApproval],
- [Status::Build::Cancelable,
+ [Status::Build::WaitingForResource,
+ Status::Build::Cancelable,
Status::Build::Retryable],
[Status::Build::FailedUnmetPrerequisites,
Status::Build::Failed],
diff --git a/lib/gitlab/ci/status/composite.rb b/lib/gitlab/ci/status/composite.rb
index e854164d377..1ba78b357e5 100644
--- a/lib/gitlab/ci/status/composite.rb
+++ b/lib/gitlab/ci/status/composite.rb
@@ -7,17 +7,19 @@ module Gitlab
include Gitlab::Utils::StrongMemoize
# This class accepts an array of arrays/hashes/or objects
- def initialize(all_statuses, with_allow_failure: true, dag: false)
- unless all_statuses.respond_to?(:pluck)
- raise ArgumentError, "all_statuses needs to respond to `.pluck`"
+ # `with_allow_failure` will be removed when deleting ci_remove_ensure_stage_service
+ def initialize(all_jobs, with_allow_failure: true, dag: false, project: nil)
+ unless all_jobs.respond_to?(:pluck)
+ raise ArgumentError, "all_jobs needs to respond to `.pluck`"
end
@status_set = Set.new
@status_key = 0
@allow_failure_key = 1 if with_allow_failure
@dag = dag
+ @project = project
- consume_all_statuses(all_statuses)
+ consume_all_jobs(all_jobs)
end
# The status calculation is order dependent,
@@ -26,6 +28,14 @@ module Gitlab
# 2. In other cases we assume that status is of that type
# based on what statuses are no longer valid based on the
# data set that we have
+ #
+ # This method is used for three cases:
+ # 1. When it is called for a stage or a pipeline (with `all_jobs` from all jobs in a stage or a pipeline),
+ # then, the returned status is assigned to the stage or pipeline.
+ # 2. When it is called for a job (with `all_jobs` from all previous jobs or all needed jobs),
+ # then, the returned status is used to determine if the job is processed or not.
+ # 3. When it is called for a group (of jobs that are related),
+ # then, the returned status is used to show the overall status of the group.
# rubocop: disable Metrics/CyclomaticComplexity
# rubocop: disable Metrics/PerceivedComplexity
def status
@@ -35,9 +45,6 @@ module Gitlab
if @dag && any_skipped_or_ignored?
# The DAG job is skipped if one of the needs does not run at all.
'skipped'
- elsif @dag && !only_of?(:success, :failed, :canceled, :skipped, :success_with_warnings)
- # DAG is blocked from executing if a dependent is not "complete"
- 'pending'
elsif only_of?(:skipped, :ignored)
'skipped'
elsif only_of?(:success, :skipped, :success_with_warnings, :ignored)
@@ -94,42 +101,41 @@ module Gitlab
any_of?(:skipped) || any_of?(:ignored)
end
- def consume_all_statuses(all_statuses)
+ def consume_all_jobs(all_jobs)
columns = []
columns[@status_key] = :status
columns[@allow_failure_key] = :allow_failure if @allow_failure_key
- all_statuses
+ all_jobs
.pluck(*columns) # rubocop: disable CodeReuse/ActiveRecord
- .each(&method(:consume_status))
+ .each do |job_attrs|
+ consume_job_status(Array.wrap(job_attrs))
+ end
end
- def consume_status(description)
- # convert `"status"` into `["status"]`
- description = Array(description)
-
- status =
- if success_with_warnings?(description)
+ def consume_job_status(job_attrs)
+ status_result =
+ if success_with_warnings?(job_attrs)
:success_with_warnings
- elsif ignored_status?(description)
+ elsif ignored_status?(job_attrs)
:ignored
else
- description[@status_key].to_sym
+ job_attrs[@status_key].to_sym
end
- @status_set.add(status)
+ @status_set.add(status_result)
end
- def success_with_warnings?(status)
+ def success_with_warnings?(job_attrs)
@allow_failure_key &&
- status[@allow_failure_key] &&
- ::Ci::HasStatus::PASSED_WITH_WARNINGS_STATUSES.include?(status[@status_key])
+ job_attrs[@allow_failure_key] &&
+ ::Ci::HasStatus::PASSED_WITH_WARNINGS_STATUSES.include?(job_attrs[@status_key])
end
- def ignored_status?(status)
+ def ignored_status?(job_attrs)
@allow_failure_key &&
- status[@allow_failure_key] &&
- ::Ci::HasStatus::EXCLUDE_IGNORED_STATUSES.include?(status[@status_key])
+ job_attrs[@allow_failure_key] &&
+ ::Ci::HasStatus::IGNORED_STATUSES.include?(job_attrs[@status_key])
end
end
end
diff --git a/lib/gitlab/ci/status/processable/waiting_for_resource.rb b/lib/gitlab/ci/status/processable/waiting_for_resource.rb
index c9b1dd795d0..ac82c99b5f1 100644
--- a/lib/gitlab/ci/status/processable/waiting_for_resource.rb
+++ b/lib/gitlab/ci/status/processable/waiting_for_resource.rb
@@ -17,9 +17,39 @@ module Gitlab
}
end
+ def has_action?
+ current_processable.present?
+ end
+
+ def action_icon
+ nil
+ end
+
+ def action_title
+ nil
+ end
+
+ def action_button_title
+ _('View job currently using resource')
+ end
+
+ def action_path
+ project_job_path(subject.project, current_processable)
+ end
+
+ def action_method
+ :get
+ end
+
def self.matches?(processable, _)
processable.waiting_for_resource?
end
+
+ private
+
+ def current_processable
+ @current_processable ||= subject.resource_group.current_processable
+ end
end
end
end
diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml
index 11420b05dfb..4f12f0cd3b8 100644
--- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml
@@ -58,7 +58,6 @@ variables:
POSTGRES_USER: user
POSTGRES_PASSWORD: testing-password
- POSTGRES_ENABLED: "true"
POSTGRES_DB: $CI_ENVIRONMENT_SLUG
DOCKER_DRIVER: overlay2
diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml
index 40f5109851b..7a4c65f8c5b 100644
--- a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml
@@ -1,5 +1,5 @@
variables:
- AUTO_BUILD_IMAGE_VERSION: 'v1.28.0'
+ AUTO_BUILD_IMAGE_VERSION: 'v1.32.0'
build:
stage: build
diff --git a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml
index 40f5109851b..7a4c65f8c5b 100644
--- a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml
@@ -1,5 +1,5 @@
variables:
- AUTO_BUILD_IMAGE_VERSION: 'v1.28.0'
+ AUTO_BUILD_IMAGE_VERSION: 'v1.32.0'
build:
stage: build
diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml
index 47b79302828..b2ab6704e35 100644
--- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml
@@ -8,7 +8,7 @@ code_quality:
variables:
DOCKER_DRIVER: overlay2
DOCKER_TLS_CERTDIR: ""
- CODE_QUALITY_IMAGE_TAG: "0.89.0"
+ CODE_QUALITY_IMAGE_TAG: "0.94.0"
CODE_QUALITY_IMAGE: "$CI_TEMPLATE_REGISTRY_HOST/gitlab-org/ci-cd/codequality:$CODE_QUALITY_IMAGE_TAG"
needs: []
script:
diff --git a/lib/gitlab/ci/templates/Jobs/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Container-Scanning.gitlab-ci.yml
index fa609afc5a8..192d06bfa14 100644
--- a/lib/gitlab/ci/templates/Jobs/Container-Scanning.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Container-Scanning.gitlab-ci.yml
@@ -22,7 +22,8 @@
# List of available variables: https://docs.gitlab.com/ee/user/application_security/container_scanning/#available-variables
variables:
- CS_ANALYZER_IMAGE: "$CI_TEMPLATE_REGISTRY_HOST/security-products/container-scanning:5"
+ CS_ANALYZER_IMAGE: "$CI_TEMPLATE_REGISTRY_HOST/security-products/container-scanning:6"
+ CS_SCHEMA_MODEL: 15
container_scanning:
image: "$CS_ANALYZER_IMAGE$CS_IMAGE_SUFFIX"
@@ -39,12 +40,12 @@ container_scanning:
reports:
container_scanning: gl-container-scanning-report.json
dependency_scanning: gl-dependency-scanning-report.json
- paths: [gl-container-scanning-report.json, gl-dependency-scanning-report.json]
+ paths: [gl-container-scanning-report.json, gl-dependency-scanning-report.json, "**/gl-sbom-*.cdx.json"]
dependencies: []
script:
- gtcs scan
rules:
- - if: $CONTAINER_SCANNING_DISABLED
+ - if: $CONTAINER_SCANNING_DISABLED == 'true' || $CONTAINER_SCANNING_DISABLED == '1'
when: never
- if: $CI_COMMIT_BRANCH &&
$CI_GITLAB_FIPS_MODE == "true" &&
diff --git a/lib/gitlab/ci/templates/Jobs/Container-Scanning.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Container-Scanning.latest.gitlab-ci.yml
index f750bda2a3f..9a4c75e7402 100644
--- a/lib/gitlab/ci/templates/Jobs/Container-Scanning.latest.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Container-Scanning.latest.gitlab-ci.yml
@@ -22,7 +22,8 @@
# List of available variables: https://docs.gitlab.com/ee/user/application_security/container_scanning/#available-variables
variables:
- CS_ANALYZER_IMAGE: "$CI_TEMPLATE_REGISTRY_HOST/security-products/container-scanning:5"
+ CS_ANALYZER_IMAGE: "$CI_TEMPLATE_REGISTRY_HOST/security-products/container-scanning:6"
+ CS_SCHEMA_MODEL: 15
container_scanning:
image: "$CS_ANALYZER_IMAGE$CS_IMAGE_SUFFIX"
@@ -39,12 +40,12 @@ container_scanning:
reports:
container_scanning: gl-container-scanning-report.json
dependency_scanning: gl-dependency-scanning-report.json
- paths: [gl-container-scanning-report.json, gl-dependency-scanning-report.json]
+ paths: [gl-container-scanning-report.json, gl-dependency-scanning-report.json, "**/gl-sbom-*.cdx.json"]
dependencies: []
script:
- gtcs scan
rules:
- - if: $CONTAINER_SCANNING_DISABLED
+ - if: $CONTAINER_SCANNING_DISABLED == 'true' || $CONTAINER_SCANNING_DISABLED == '1'
when: never
# Add the job to merge request pipelines if there's an open merge request.
diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml
index aa2356f6a34..4ee5fa74df9 100644
--- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml
@@ -1,5 +1,5 @@
variables:
- DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.46.0'
+ DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.48.2'
.dast-auto-deploy:
image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${DAST_AUTO_DEPLOY_IMAGE_VERSION}"
diff --git a/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml
index eb8e5de5b56..63cf265fc6e 100644
--- a/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml
@@ -14,7 +14,7 @@ variables:
SECURE_ANALYZERS_PREFIX: "$CI_TEMPLATE_REGISTRY_HOST/security-products"
DS_EXCLUDED_ANALYZERS: ""
DS_EXCLUDED_PATHS: "spec, test, tests, tmp"
- DS_MAJOR_VERSION: 3
+ DS_MAJOR_VERSION: 4
DS_SCHEMA_MODEL: 15
dependency_scanning:
@@ -56,15 +56,16 @@ dependency_scanning:
.gemnasium-shared-rule:
exists:
- - '{Gemfile.lock,*/Gemfile.lock,*/*/Gemfile.lock}'
- - '{composer.lock,*/composer.lock,*/*/composer.lock}'
- - '{gems.locked,*/gems.locked,*/*/gems.locked}'
- - '{go.sum,*/go.sum,*/*/go.sum}'
- - '{npm-shrinkwrap.json,*/npm-shrinkwrap.json,*/*/npm-shrinkwrap.json}'
- - '{package-lock.json,*/package-lock.json,*/*/package-lock.json}'
- - '{yarn.lock,*/yarn.lock,*/*/yarn.lock}'
- - '{packages.lock.json,*/packages.lock.json,*/*/packages.lock.json}'
- - '{conan.lock,*/conan.lock,*/*/conan.lock}'
+ - '**/Gemfile.lock'
+ - '**/composer.lock'
+ - '**/gems.locked'
+ - '**/go.sum'
+ - '**/npm-shrinkwrap.json'
+ - '**/package-lock.json'
+ - '**/yarn.lock'
+ - '**/pnpm-lock.yaml'
+ - '**/packages.lock.json'
+ - '**/conan.lock'
gemnasium-dependency_scanning:
extends:
@@ -74,7 +75,7 @@ gemnasium-dependency_scanning:
DS_ANALYZER_NAME: "gemnasium"
GEMNASIUM_LIBRARY_SCAN_ENABLED: "true"
rules:
- - if: $DEPENDENCY_SCANNING_DISABLED
+ - if: $DEPENDENCY_SCANNING_DISABLED == 'true' || $DEPENDENCY_SCANNING_DISABLED == '1'
when: never
- if: $DS_EXCLUDED_ANALYZERS =~ /gemnasium([^-]|$)/
when: never
@@ -91,10 +92,10 @@ gemnasium-dependency_scanning:
.gemnasium-maven-shared-rule:
exists:
- - '{build.gradle,*/build.gradle,*/*/build.gradle}'
- - '{build.gradle.kts,*/build.gradle.kts,*/*/build.gradle.kts}'
- - '{build.sbt,*/build.sbt,*/*/build.sbt}'
- - '{pom.xml,*/pom.xml,*/*/pom.xml}'
+ - '**/build.gradle'
+ - '**/build.gradle.kts'
+ - '**/build.sbt'
+ - '**/pom.xml'
gemnasium-maven-dependency_scanning:
extends:
@@ -103,7 +104,7 @@ gemnasium-maven-dependency_scanning:
variables:
DS_ANALYZER_NAME: "gemnasium-maven"
rules:
- - if: $DEPENDENCY_SCANNING_DISABLED
+ - if: $DEPENDENCY_SCANNING_DISABLED == 'true' || $DEPENDENCY_SCANNING_DISABLED == '1'
when: never
- if: $DS_EXCLUDED_ANALYZERS =~ /gemnasium-maven/
when: never
@@ -119,12 +120,13 @@ gemnasium-maven-dependency_scanning:
.gemnasium-python-shared-rule:
exists:
- - '{requirements.txt,*/requirements.txt,*/*/requirements.txt}'
- - '{requirements.pip,*/requirements.pip,*/*/requirements.pip}'
- - '{Pipfile,*/Pipfile,*/*/Pipfile}'
- - '{requires.txt,*/requires.txt,*/*/requires.txt}'
- - '{setup.py,*/setup.py,*/*/setup.py}'
- - '{poetry.lock,*/poetry.lock,*/*/poetry.lock}'
+ - '**/requirements.txt'
+ - '**/requirements.pip'
+ - '**/Pipfile'
+ - '**/Pipfile.lock'
+ - '**/requires.txt'
+ - '**/setup.py'
+ - '**/poetry.lock'
gemnasium-python-dependency_scanning:
extends:
@@ -133,7 +135,7 @@ gemnasium-python-dependency_scanning:
variables:
DS_ANALYZER_NAME: "gemnasium-python"
rules:
- - if: $DEPENDENCY_SCANNING_DISABLED
+ - if: $DEPENDENCY_SCANNING_DISABLED == 'true' || $DEPENDENCY_SCANNING_DISABLED == '1'
when: never
- if: $DS_EXCLUDED_ANALYZERS =~ /gemnasium-python/
when: never
diff --git a/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.latest.gitlab-ci.yml
index 655ac6ee712..4d7c3930741 100644
--- a/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.latest.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.latest.gitlab-ci.yml
@@ -14,7 +14,7 @@ variables:
SECURE_ANALYZERS_PREFIX: "$CI_TEMPLATE_REGISTRY_HOST/security-products"
DS_EXCLUDED_ANALYZERS: ""
DS_EXCLUDED_PATHS: "spec, test, tests, tmp"
- DS_MAJOR_VERSION: 3
+ DS_MAJOR_VERSION: 4
DS_SCHEMA_MODEL: 15
dependency_scanning:
@@ -56,15 +56,16 @@ dependency_scanning:
.gemnasium-shared-rule:
exists:
- - '{Gemfile.lock,*/Gemfile.lock,*/*/Gemfile.lock}'
- - '{composer.lock,*/composer.lock,*/*/composer.lock}'
- - '{gems.locked,*/gems.locked,*/*/gems.locked}'
- - '{go.sum,*/go.sum,*/*/go.sum}'
- - '{npm-shrinkwrap.json,*/npm-shrinkwrap.json,*/*/npm-shrinkwrap.json}'
- - '{package-lock.json,*/package-lock.json,*/*/package-lock.json}'
- - '{yarn.lock,*/yarn.lock,*/*/yarn.lock}'
- - '{packages.lock.json,*/packages.lock.json,*/*/packages.lock.json}'
- - '{conan.lock,*/conan.lock,*/*/conan.lock}'
+ - '**/Gemfile.lock'
+ - '**/composer.lock'
+ - '**/gems.locked'
+ - '**/go.sum'
+ - '**/npm-shrinkwrap.json'
+ - '**/package-lock.json'
+ - '**/yarn.lock'
+ - '**/pnpm-lock.yaml'
+ - '**/packages.lock.json'
+ - '**/conan.lock'
gemnasium-dependency_scanning:
extends:
@@ -74,7 +75,7 @@ gemnasium-dependency_scanning:
DS_ANALYZER_NAME: "gemnasium"
GEMNASIUM_LIBRARY_SCAN_ENABLED: "true"
rules:
- - if: $DEPENDENCY_SCANNING_DISABLED
+ - if: $DEPENDENCY_SCANNING_DISABLED == 'true' || $DEPENDENCY_SCANNING_DISABLED == '1'
when: never
- if: $DS_EXCLUDED_ANALYZERS =~ /gemnasium([^-]|$)/
when: never
@@ -109,10 +110,10 @@ gemnasium-dependency_scanning:
.gemnasium-maven-shared-rule:
exists:
- - '{build.gradle,*/build.gradle,*/*/build.gradle}'
- - '{build.gradle.kts,*/build.gradle.kts,*/*/build.gradle.kts}'
- - '{build.sbt,*/build.sbt,*/*/build.sbt}'
- - '{pom.xml,*/pom.xml,*/*/pom.xml}'
+ - '**/build.gradle'
+ - '**/build.gradle.kts'
+ - '**/build.sbt'
+ - '**/pom.xml'
gemnasium-maven-dependency_scanning:
extends:
@@ -121,7 +122,7 @@ gemnasium-maven-dependency_scanning:
variables:
DS_ANALYZER_NAME: "gemnasium-maven"
rules:
- - if: $DEPENDENCY_SCANNING_DISABLED
+ - if: $DEPENDENCY_SCANNING_DISABLED == 'true' || $DEPENDENCY_SCANNING_DISABLED == '1'
when: never
- if: $DS_EXCLUDED_ANALYZERS =~ /gemnasium-maven/
when: never
@@ -155,12 +156,13 @@ gemnasium-maven-dependency_scanning:
.gemnasium-python-shared-rule:
exists:
- - '{requirements.txt,*/requirements.txt,*/*/requirements.txt}'
- - '{requirements.pip,*/requirements.pip,*/*/requirements.pip}'
- - '{Pipfile,*/Pipfile,*/*/Pipfile}'
- - '{requires.txt,*/requires.txt,*/*/requires.txt}'
- - '{setup.py,*/setup.py,*/*/setup.py}'
- - '{poetry.lock,*/poetry.lock,*/*/poetry.lock}'
+ - '**/requirements.txt'
+ - '**/requirements.pip'
+ - '**/Pipfile'
+ - '**/Pipfile.lock'
+ - '**/requires.txt'
+ - '**/setup.py'
+ - '**/poetry.lock'
gemnasium-python-dependency_scanning:
extends:
@@ -169,7 +171,7 @@ gemnasium-python-dependency_scanning:
variables:
DS_ANALYZER_NAME: "gemnasium-python"
rules:
- - if: $DEPENDENCY_SCANNING_DISABLED
+ - if: $DEPENDENCY_SCANNING_DISABLED == 'true' || $DEPENDENCY_SCANNING_DISABLED == '1'
when: never
- if: $DS_EXCLUDED_ANALYZERS =~ /gemnasium-python/
when: never
diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml
index 372b782c0a0..622b44d78ad 100644
--- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml
@@ -1,5 +1,5 @@
variables:
- AUTO_DEPLOY_IMAGE_VERSION: 'v2.46.0'
+ AUTO_DEPLOY_IMAGE_VERSION: 'v2.48.2'
.auto-deploy:
image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}"
diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml
index feba2efcf22..2954ddf8a35 100644
--- a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml
@@ -1,5 +1,5 @@
variables:
- AUTO_DEPLOY_IMAGE_VERSION: 'v2.46.0'
+ AUTO_DEPLOY_IMAGE_VERSION: 'v2.48.2'
.auto-deploy:
image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}"
diff --git a/lib/gitlab/ci/templates/Jobs/License-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/License-Scanning.gitlab-ci.yml
index f8668699fe5..b1c81e9ed5b 100644
--- a/lib/gitlab/ci/templates/Jobs/License-Scanning.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/License-Scanning.gitlab-ci.yml
@@ -32,7 +32,7 @@ license_scanning:
license_scanning: gl-license-scanning-report.json
dependencies: []
rules:
- - if: $LICENSE_MANAGEMENT_DISABLED
+ - if: $LICENSE_MANAGEMENT_DISABLED == 'true' || $LICENSE_MANAGEMENT_DISABLED == '1'
when: never
- if: $CI_COMMIT_BRANCH &&
$GITLAB_FEATURES =~ /\blicense_scanning\b/
diff --git a/lib/gitlab/ci/templates/Jobs/License-Scanning.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/License-Scanning.latest.gitlab-ci.yml
index e47f669c2e2..8e1b0159cb0 100644
--- a/lib/gitlab/ci/templates/Jobs/License-Scanning.latest.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/License-Scanning.latest.gitlab-ci.yml
@@ -32,7 +32,7 @@ license_scanning:
license_scanning: gl-license-scanning-report.json
dependencies: []
rules:
- - if: $LICENSE_MANAGEMENT_DISABLED
+ - if: $LICENSE_MANAGEMENT_DISABLED == 'true' || $LICENSE_MANAGEMENT_DISABLED == '1'
when: never
# Add the job to merge request pipelines if there's an open merge request.
diff --git a/lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml
index 12105e0e95d..a849d36a5b8 100644
--- a/lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml
@@ -6,7 +6,7 @@ load_performance:
DOCKER_TLS_CERTDIR: ""
K6_IMAGE: grafana/k6
K6_VERSION: 0.41.0
- K6_TEST_FILE: raw.githubusercontent.com/grafana/k6/master/samples/http_get.js
+ K6_TEST_FILE: raw.githubusercontent.com/grafana/k6/master/examples/http_get.js
K6_OPTIONS: ''
K6_DOCKER_OPTIONS: ''
services:
diff --git a/lib/gitlab/ci/templates/Jobs/SAST-IaC.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST-IaC.gitlab-ci.yml
index c195ecd8ee5..a64e1e4a40f 100644
--- a/lib/gitlab/ci/templates/Jobs/SAST-IaC.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/SAST-IaC.gitlab-ci.yml
@@ -31,10 +31,10 @@ kics-iac-sast:
image:
name: "$SAST_ANALYZER_IMAGE"
variables:
- SAST_ANALYZER_IMAGE_TAG: 3
+ SAST_ANALYZER_IMAGE_TAG: 4
SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/kics:$SAST_ANALYZER_IMAGE_TAG$SAST_IMAGE_SUFFIX"
rules:
- - if: $SAST_DISABLED
+ - if: $SAST_DISABLED == 'true' || $SAST_DISABLED == '1'
when: never
- if: $SAST_EXCLUDED_ANALYZERS =~ /kics/
when: never
diff --git a/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml
index 77048037915..77f2c5a8c99 100644
--- a/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml
@@ -31,10 +31,10 @@ kics-iac-sast:
image:
name: "$SAST_ANALYZER_IMAGE"
variables:
- SAST_ANALYZER_IMAGE_TAG: 3
+ SAST_ANALYZER_IMAGE_TAG: 4
SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/kics:$SAST_ANALYZER_IMAGE_TAG$SAST_IMAGE_SUFFIX"
rules:
- - if: $SAST_DISABLED
+ - if: $SAST_DISABLED == 'true' || $SAST_DISABLED == '1'
when: never
- if: $SAST_EXCLUDED_ANALYZERS =~ /kics/
when: never
diff --git a/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml
index 8b49d2de8cf..d567ab2a141 100644
--- a/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml
@@ -1,7 +1,7 @@
# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/sast/
#
# Configure SAST with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html).
-# List of available variables: https://docs.gitlab.com/ee/user/application_security/sast/index.html#available-variables
+# List of available variables: https://docs.gitlab.com/ee/user/application_security/sast/index.html#available-cicd-variables
variables:
# Setting this variable will affect all Security templates
@@ -48,10 +48,10 @@ brakeman-sast:
image:
name: "$SAST_ANALYZER_IMAGE"
variables:
- SAST_ANALYZER_IMAGE_TAG: 3
+ SAST_ANALYZER_IMAGE_TAG: 4
SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/brakeman:$SAST_ANALYZER_IMAGE_TAG"
rules:
- - if: $SAST_DISABLED
+ - if: $SAST_DISABLED == 'true' || $SAST_DISABLED == '1'
when: never
- if: $SAST_EXCLUDED_ANALYZERS =~ /brakeman/
when: never
@@ -74,10 +74,10 @@ flawfinder-sast:
image:
name: "$SAST_ANALYZER_IMAGE"
variables:
- SAST_ANALYZER_IMAGE_TAG: 3
+ SAST_ANALYZER_IMAGE_TAG: 4
SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/flawfinder:$SAST_ANALYZER_IMAGE_TAG"
rules:
- - if: $SAST_DISABLED
+ - if: $SAST_DISABLED == 'true' || $SAST_DISABLED == '1'
when: never
- if: $SAST_EXCLUDED_ANALYZERS =~ /flawfinder/
when: never
@@ -95,10 +95,10 @@ kubesec-sast:
image:
name: "$SAST_ANALYZER_IMAGE"
variables:
- SAST_ANALYZER_IMAGE_TAG: 3
+ SAST_ANALYZER_IMAGE_TAG: 4
SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/kubesec:$SAST_ANALYZER_IMAGE_TAG"
rules:
- - if: $SAST_DISABLED
+ - if: $SAST_DISABLED == 'true' || $SAST_DISABLED == '1'
when: never
- if: $SAST_EXCLUDED_ANALYZERS =~ /kubesec/
when: never
@@ -119,13 +119,13 @@ gosec-sast:
image:
name: "$SAST_ANALYZER_IMAGE"
variables:
- SAST_ANALYZER_IMAGE_TAG: 3
+ SAST_ANALYZER_IMAGE_TAG: 4
SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/mobsf:$SAST_ANALYZER_IMAGE_TAG"
mobsf-android-sast:
extends: .mobsf-sast
rules:
- - if: $SAST_DISABLED
+ - if: $SAST_DISABLED == 'true' || $SAST_DISABLED == '1'
when: never
- if: $SAST_EXCLUDED_ANALYZERS =~ /mobsf/
when: never
@@ -138,7 +138,7 @@ mobsf-android-sast:
mobsf-ios-sast:
extends: .mobsf-sast
rules:
- - if: $SAST_DISABLED
+ - if: $SAST_DISABLED == 'true' || $SAST_DISABLED == '1'
when: never
- if: $SAST_EXCLUDED_ANALYZERS =~ /mobsf/
when: never
@@ -153,10 +153,10 @@ nodejs-scan-sast:
image:
name: "$SAST_ANALYZER_IMAGE"
variables:
- SAST_ANALYZER_IMAGE_TAG: 3
+ SAST_ANALYZER_IMAGE_TAG: 4
SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/nodejs-scan:$SAST_ANALYZER_IMAGE_TAG"
rules:
- - if: $SAST_DISABLED
+ - if: $SAST_DISABLED == 'true' || $SAST_DISABLED == '1'
when: never
- if: $SAST_EXCLUDED_ANALYZERS =~ /nodejs-scan/
when: never
@@ -169,10 +169,10 @@ phpcs-security-audit-sast:
image:
name: "$SAST_ANALYZER_IMAGE"
variables:
- SAST_ANALYZER_IMAGE_TAG: 3
+ SAST_ANALYZER_IMAGE_TAG: 4
SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/phpcs-security-audit:$SAST_ANALYZER_IMAGE_TAG"
rules:
- - if: $SAST_DISABLED
+ - if: $SAST_DISABLED == 'true' || $SAST_DISABLED == '1'
when: never
- if: $SAST_EXCLUDED_ANALYZERS =~ /phpcs-security-audit/
when: never
@@ -185,10 +185,10 @@ pmd-apex-sast:
image:
name: "$SAST_ANALYZER_IMAGE"
variables:
- SAST_ANALYZER_IMAGE_TAG: 3
+ SAST_ANALYZER_IMAGE_TAG: 4
SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/pmd-apex:$SAST_ANALYZER_IMAGE_TAG"
rules:
- - if: $SAST_DISABLED
+ - if: $SAST_DISABLED == 'true' || $SAST_DISABLED == '1'
when: never
- if: $SAST_EXCLUDED_ANALYZERS =~ /pmd-apex/
when: never
@@ -198,20 +198,12 @@ pmd-apex-sast:
security-code-scan-sast:
extends: .sast-analyzer
- image:
- name: "$SAST_ANALYZER_IMAGE"
- variables:
- SAST_ANALYZER_IMAGE_TAG: '3'
- SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/security-code-scan:$SAST_ANALYZER_IMAGE_TAG"
+ script:
+ - echo "This job was deprecated in GitLab 15.9 and removed in GitLab 16.0"
+ - echo "For more information see https://gitlab.com/gitlab-org/gitlab/-/issues/390416"
+ - exit 1
rules:
- - if: $SAST_DISABLED
- when: never
- - if: $SAST_EXCLUDED_ANALYZERS =~ /security-code-scan/
- when: never
- - if: $CI_COMMIT_BRANCH
- exists:
- - '**/*.csproj'
- - '**/*.vbproj'
+ - when: never
semgrep-sast:
extends: .sast-analyzer
@@ -219,10 +211,10 @@ semgrep-sast:
name: "$SAST_ANALYZER_IMAGE"
variables:
SEARCH_MAX_DEPTH: 20
- SAST_ANALYZER_IMAGE_TAG: 3
+ SAST_ANALYZER_IMAGE_TAG: 4
SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/semgrep:$SAST_ANALYZER_IMAGE_TAG$SAST_IMAGE_SUFFIX"
rules:
- - if: $SAST_DISABLED
+ - if: $SAST_DISABLED == 'true' || $SAST_DISABLED == '1'
when: never
- if: $SAST_EXCLUDED_ANALYZERS =~ /semgrep/
when: never
@@ -246,10 +238,10 @@ sobelow-sast:
image:
name: "$SAST_ANALYZER_IMAGE"
variables:
- SAST_ANALYZER_IMAGE_TAG: 3
+ SAST_ANALYZER_IMAGE_TAG: 4
SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/sobelow:$SAST_ANALYZER_IMAGE_TAG"
rules:
- - if: $SAST_DISABLED
+ - if: $SAST_DISABLED == 'true' || $SAST_DISABLED == '1'
when: never
- if: $SAST_EXCLUDED_ANALYZERS =~ /sobelow/
when: never
@@ -262,7 +254,7 @@ spotbugs-sast:
image:
name: "$SAST_ANALYZER_IMAGE"
variables:
- SAST_ANALYZER_IMAGE_TAG: 3
+ SAST_ANALYZER_IMAGE_TAG: 4
SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/spotbugs:$SAST_ANALYZER_IMAGE_TAG"
rules:
- if: $SAST_EXCLUDED_ANALYZERS =~ /spotbugs/
@@ -271,7 +263,7 @@ spotbugs-sast:
exists:
- '**/AndroidManifest.xml'
when: never
- - if: $SAST_DISABLED
+ - if: $SAST_DISABLED == 'true' || $SAST_DISABLED == '1'
when: never
- if: $CI_COMMIT_BRANCH
exists:
diff --git a/lib/gitlab/ci/templates/Jobs/SAST.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST.latest.gitlab-ci.yml
index 1c4dbe6cd0f..88d10f8b235 100644
--- a/lib/gitlab/ci/templates/Jobs/SAST.latest.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/SAST.latest.gitlab-ci.yml
@@ -48,10 +48,10 @@ brakeman-sast:
image:
name: "$SAST_ANALYZER_IMAGE"
variables:
- SAST_ANALYZER_IMAGE_TAG: 3
+ SAST_ANALYZER_IMAGE_TAG: 4
SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/brakeman:$SAST_ANALYZER_IMAGE_TAG"
rules:
- - if: $SAST_DISABLED
+ - if: $SAST_DISABLED == 'true' || $SAST_DISABLED == '1'
when: never
- if: $SAST_EXCLUDED_ANALYZERS =~ /brakeman/
when: never
@@ -80,10 +80,10 @@ flawfinder-sast:
image:
name: "$SAST_ANALYZER_IMAGE"
variables:
- SAST_ANALYZER_IMAGE_TAG: 3
+ SAST_ANALYZER_IMAGE_TAG: 4
SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/flawfinder:$SAST_ANALYZER_IMAGE_TAG"
rules:
- - if: $SAST_DISABLED
+ - if: $SAST_DISABLED == 'true' || $SAST_DISABLED == '1'
when: never
- if: $SAST_EXCLUDED_ANALYZERS =~ /flawfinder/
when: never
@@ -120,10 +120,10 @@ kubesec-sast:
image:
name: "$SAST_ANALYZER_IMAGE"
variables:
- SAST_ANALYZER_IMAGE_TAG: 3
+ SAST_ANALYZER_IMAGE_TAG: 4
SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/kubesec:$SAST_ANALYZER_IMAGE_TAG"
rules:
- - if: $SAST_DISABLED
+ - if: $SAST_DISABLED == 'true' || $SAST_DISABLED == '1'
when: never
- if: $SAST_EXCLUDED_ANALYZERS =~ /kubesec/
when: never
@@ -141,13 +141,13 @@ kubesec-sast:
image:
name: "$SAST_ANALYZER_IMAGE"
variables:
- SAST_ANALYZER_IMAGE_TAG: 3
+ SAST_ANALYZER_IMAGE_TAG: 4
SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/mobsf:$SAST_ANALYZER_IMAGE_TAG"
mobsf-android-sast:
extends: .mobsf-sast
rules:
- - if: $SAST_DISABLED
+ - if: $SAST_DISABLED == 'true' || $SAST_DISABLED == '1'
when: never
- if: $SAST_EXCLUDED_ANALYZERS =~ /mobsf/
when: never
@@ -169,7 +169,7 @@ mobsf-android-sast:
mobsf-ios-sast:
extends: .mobsf-sast
rules:
- - if: $SAST_DISABLED
+ - if: $SAST_DISABLED == 'true' || $SAST_DISABLED == '1'
when: never
- if: $SAST_EXCLUDED_ANALYZERS =~ /mobsf/
when: never
@@ -193,10 +193,10 @@ nodejs-scan-sast:
image:
name: "$SAST_ANALYZER_IMAGE"
variables:
- SAST_ANALYZER_IMAGE_TAG: 3
+ SAST_ANALYZER_IMAGE_TAG: 4
SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/nodejs-scan:$SAST_ANALYZER_IMAGE_TAG"
rules:
- - if: $SAST_DISABLED
+ - if: $SAST_DISABLED == 'true' || $SAST_DISABLED == '1'
when: never
- if: $SAST_EXCLUDED_ANALYZERS =~ /nodejs-scan/
when: never
@@ -214,10 +214,10 @@ phpcs-security-audit-sast:
image:
name: "$SAST_ANALYZER_IMAGE"
variables:
- SAST_ANALYZER_IMAGE_TAG: 3
+ SAST_ANALYZER_IMAGE_TAG: 4
SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/phpcs-security-audit:$SAST_ANALYZER_IMAGE_TAG"
rules:
- - if: $SAST_DISABLED
+ - if: $SAST_DISABLED == 'true' || $SAST_DISABLED == '1'
when: never
- if: $SAST_EXCLUDED_ANALYZERS =~ /phpcs-security-audit/
when: never
@@ -235,10 +235,10 @@ pmd-apex-sast:
image:
name: "$SAST_ANALYZER_IMAGE"
variables:
- SAST_ANALYZER_IMAGE_TAG: 3
+ SAST_ANALYZER_IMAGE_TAG: 4
SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/pmd-apex:$SAST_ANALYZER_IMAGE_TAG"
rules:
- - if: $SAST_DISABLED
+ - if: $SAST_DISABLED == 'true' || $SAST_DISABLED == '1'
when: never
- if: $SAST_EXCLUDED_ANALYZERS =~ /pmd-apex/
when: never
@@ -253,26 +253,12 @@ pmd-apex-sast:
security-code-scan-sast:
extends: .sast-analyzer
- image:
- name: "$SAST_ANALYZER_IMAGE"
- variables:
- SAST_ANALYZER_IMAGE_TAG: 3
- SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/security-code-scan:$SAST_ANALYZER_IMAGE_TAG"
+ script:
+ - echo "This job was deprecated in GitLab 15.9 and removed in GitLab 16.0"
+ - echo "For more information see https://gitlab.com/gitlab-org/gitlab/-/issues/390416"
+ - exit 1
rules:
- - if: $SAST_DISABLED
- when: never
- - if: $SAST_EXCLUDED_ANALYZERS =~ /security-code-scan/
- when: never
- - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request.
- exists:
- - '**/*.csproj'
- - '**/*.vbproj'
- - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline.
- when: never
- - if: $CI_COMMIT_BRANCH # If there's no open merge request, add it to a *branch* pipeline instead.
- exists:
- - '**/*.csproj'
- - '**/*.vbproj'
+ - when: never
semgrep-sast:
extends: .sast-analyzer
@@ -280,10 +266,10 @@ semgrep-sast:
name: "$SAST_ANALYZER_IMAGE"
variables:
SEARCH_MAX_DEPTH: 20
- SAST_ANALYZER_IMAGE_TAG: 3
+ SAST_ANALYZER_IMAGE_TAG: 4
SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/semgrep:$SAST_ANALYZER_IMAGE_TAG$SAST_IMAGE_SUFFIX"
rules:
- - if: $SAST_DISABLED
+ - if: $SAST_DISABLED == 'true' || $SAST_DISABLED == '1'
when: never
- if: $SAST_EXCLUDED_ANALYZERS =~ /semgrep/
when: never
@@ -323,10 +309,10 @@ sobelow-sast:
image:
name: "$SAST_ANALYZER_IMAGE"
variables:
- SAST_ANALYZER_IMAGE_TAG: 3
+ SAST_ANALYZER_IMAGE_TAG: 4
SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/sobelow:$SAST_ANALYZER_IMAGE_TAG"
rules:
- - if: $SAST_DISABLED
+ - if: $SAST_DISABLED == 'true' || $SAST_DISABLED == '1'
when: never
- if: $SAST_EXCLUDED_ANALYZERS =~ /sobelow/
when: never
@@ -344,7 +330,7 @@ spotbugs-sast:
image:
name: "$SAST_ANALYZER_IMAGE"
variables:
- SAST_ANALYZER_IMAGE_TAG: 3
+ SAST_ANALYZER_IMAGE_TAG: 4
SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/spotbugs:$SAST_ANALYZER_IMAGE_TAG"
rules:
- if: $SAST_EXCLUDED_ANALYZERS =~ /spotbugs/
@@ -353,7 +339,7 @@ spotbugs-sast:
exists:
- '**/AndroidManifest.xml'
when: never
- - if: $SAST_DISABLED
+ - if: $SAST_DISABLED == 'true' || $SAST_DISABLED == '1'
when: never
- if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request.
exists:
diff --git a/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml
index b7a9dbf7bc6..9d0b904117a 100644
--- a/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml
@@ -8,7 +8,7 @@ variables:
SECURE_ANALYZERS_PREFIX: "$CI_TEMPLATE_REGISTRY_HOST/security-products"
SECRET_DETECTION_IMAGE_SUFFIX: ""
- SECRETS_ANALYZER_VERSION: "4"
+ SECRETS_ANALYZER_VERSION: "5"
SECRET_DETECTION_EXCLUDED_PATHS: ""
.secret-analyzer:
@@ -27,7 +27,7 @@ variables:
secret_detection:
extends: .secret-analyzer
rules:
- - if: $SECRET_DETECTION_DISABLED
+ - if: $SECRET_DETECTION_DISABLED == 'true' || $SECRET_DETECTION_DISABLED == '1'
when: never
- if: $CI_COMMIT_BRANCH
script:
diff --git a/lib/gitlab/ci/templates/Jobs/Secret-Detection.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Secret-Detection.latest.gitlab-ci.yml
index 6603ee4268e..56a8ad794dc 100644
--- a/lib/gitlab/ci/templates/Jobs/Secret-Detection.latest.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Secret-Detection.latest.gitlab-ci.yml
@@ -8,7 +8,7 @@ variables:
SECURE_ANALYZERS_PREFIX: "$CI_TEMPLATE_REGISTRY_HOST/security-products"
SECRET_DETECTION_IMAGE_SUFFIX: ""
- SECRETS_ANALYZER_VERSION: "4"
+ SECRETS_ANALYZER_VERSION: "5"
SECRET_DETECTION_EXCLUDED_PATHS: ""
.secret-analyzer:
@@ -27,7 +27,7 @@ variables:
secret_detection:
extends: .secret-analyzer
rules:
- - if: $SECRET_DETECTION_DISABLED
+ - if: $SECRET_DETECTION_DISABLED == 'true' || $SECRET_DETECTION_DISABLED == '1'
when: never
- if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request.
- if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline.
diff --git a/lib/gitlab/ci/templates/Python.gitlab-ci.yml b/lib/gitlab/ci/templates/Python.gitlab-ci.yml
index febbb36d834..d53f3ddcad4 100644
--- a/lib/gitlab/ci/templates/Python.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Python.gitlab-ci.yml
@@ -23,26 +23,24 @@ cache:
- venv/
before_script:
- - python --version # For debugging
+ - python --version ; pip --version # For debugging
- pip install virtualenv
- virtualenv venv
- source venv/bin/activate
test:
script:
- - python setup.py test
- - pip install tox flake8 # you can also use tox
- - tox -e py36,flake8
+ - pip install ruff tox # you can also use tox
+ - pip install --editable ".[test]"
+ - tox -e py,ruff
run:
script:
- - python setup.py bdist_wheel
- # an alternative approach is to install and run:
- - pip install dist/*
+ - pip install .
# run the command here
artifacts:
paths:
- - dist/*.whl
+ - build/*
pages:
script:
diff --git a/lib/gitlab/ci/templates/Security/API-Discovery.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Discovery.gitlab-ci.yml
new file mode 100644
index 00000000000..d9bc76dad1e
--- /dev/null
+++ b/lib/gitlab/ci/templates/Security/API-Discovery.gitlab-ci.yml
@@ -0,0 +1,66 @@
+# To contribute improvements to CI/CD templates, please follow the Development guide at:
+# https://docs.gitlab.com/ee/development/cicd/templates.html
+# This specific template is located at:
+# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/API-Discovery.gitlab-ci.yml
+
+# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/api_discovery/
+#
+# Configure API Discovery with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html).
+# List of available variables: https://docs.gitlab.com/ee/user/application_security/api_discovery/#available-cicd-variables
+
+variables:
+ API_DISCOVERY_PACKAGES: "$CI_API_V4_URL/projects/42503323/packages"
+ API_DISCOVERY_VERSION: "1"
+
+.api_discovery_java_spring_boot:
+ stage: test
+ allow_failure: true
+ script:
+ #
+ # Check configuration
+ - if [[ -z "$API_DISCOVERY_VERSION" ]]; then echo "Error, API_DISCOVERY_VERSION not provided. Please set this variable and re-run the pipeline."; exit 1; fi
+ #
+ # Check for required commands
+ - requires() { command -v "$1" >/dev/null 2>&1 || { echo "'$1' is required but it's not installed. Add the needed command to the job image and retry." >&2; exit 1; } }
+ - requires 'curl'
+ - requires 'java'
+ #
+ # Set JAVA_HOME if API_DISCOVERY_JAVA_HOME provided
+ - if [[ -n "$API_DISCOVERY_JAVA_HOME" ]]; then export JAVA_HOME="$API_DISCOVERY_JAVA_HOME"; export PATH="$JAVA_HOME/bin:$PATH"; fi
+ #
+ # Download jar file
+ - if [[ -n "$API_DISCOVERY_PACKAGE_TOKEN" ]]; then echo "Using API_DISCOVERY_PACKAGE_TOKEN"; export CURL_AUTH="-H PRIVATE-TOKEN:$API_DISCOVERY_PACKAGE_TOKEN"; else export CURL_AUTH=""; fi
+ - DL_URL="$API_DISCOVERY_PACKAGES/maven/com/gitlab/analyzers/api-discovery/api-discovery_spring-boot/$API_DISCOVERY_VERSION/api-discovery_spring-boot-$API_DISCOVERY_VERSION.jar"
+ - echo "Downloading Discovery jar from '${DL_URL}'"
+ - CURL_CMD="curl -L ${CURL_AUTH} --write-out "%{http_code}" --output api_discovery_java_spring_boot_${API_DISCOVERY_VERSION}.jar ${DL_URL}"
+ - STATUS_CODE=$(${CURL_CMD})
+ - RC=$?
+ - if [[ $RC -ne 0 ]]; then echo "Error connecting to GitLab API, curl exit code was $RC."; echo "To diagnose, see the curl documentation- https://everything.curl.dev/usingcurl/returns"; exit 1; fi
+ - if [[ "$STATUS_CODE" != "200" ]]; then echo "Error, Unable to download api_discovery_java_spring_boot_${API_DISCOVERY_VERSION}.jar"; echo "Error, Status Code was $STATUS_CODE, but wanted 200"; exit 1; fi
+ #
+ # Run API Discovery
+ - java -jar "api_discovery_java_spring_boot_${API_DISCOVERY_VERSION}.jar"
+ #
+ # Check for expected output file
+ - if [[ ! -e "gl-api-discovery-openapi.json" ]]; then echo "Error, Unable to find gl-api-discovery-openapi.json"; exit 1; fi
+ #
+ artifacts:
+ when: always
+ paths:
+ - gl-api-discovery-openapi.json
+ - gl-*.log
+ rules:
+ - if: $API_DISCOVERY_DISABLED
+ when: never
+ - if: $API_DISCOVERY_DISABLED_FOR_DEFAULT_BRANCH &&
+ $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME
+ when: never
+ # Add the job to merge request pipelines if there's an open merge request.
+ - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+
+ # Don't add it to a *branch* pipeline if it's already in a merge request pipeline.
+ - if: $CI_OPEN_MERGE_REQUESTS
+ when: never
+
+ # Add the job to branch pipelines.
+ - if: $CI_COMMIT_BRANCH
diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml
index cdfa4556769..544aee904d5 100644
--- a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml
@@ -26,7 +26,7 @@ variables:
# (SAST, Dependency Scanning, ...)
SECURE_ANALYZERS_PREFIX: "$CI_TEMPLATE_REGISTRY_HOST/security-products"
#
- FUZZAPI_VERSION: "2"
+ FUZZAPI_VERSION: "3"
FUZZAPI_IMAGE_SUFFIX: ""
FUZZAPI_IMAGE: api-security
@@ -35,9 +35,12 @@ apifuzzer_fuzz:
image: $SECURE_ANALYZERS_PREFIX/$FUZZAPI_IMAGE:$FUZZAPI_VERSION$FUZZAPI_IMAGE_SUFFIX
allow_failure: true
rules:
- - if: $API_FUZZING_DISABLED
+ - if: $API_FUZZING_DISABLED == 'true' || $API_FUZZING_DISABLED == '1'
when: never
- - if: $API_FUZZING_DISABLED_FOR_DEFAULT_BRANCH &&
+ - if: $API_FUZZING_DISABLED_FOR_DEFAULT_BRANCH == 'true' &&
+ $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME
+ when: never
+ - if: $API_FUZZING_DISABLED_FOR_DEFAULT_BRANCH == '1' &&
$CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME
when: never
- if: $CI_COMMIT_BRANCH &&
diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml
index f12efa1db34..feaa2965339 100644
--- a/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml
@@ -26,7 +26,7 @@ variables:
# (SAST, Dependency Scanning, ...)
SECURE_ANALYZERS_PREFIX: "$CI_TEMPLATE_REGISTRY_HOST/security-products"
#
- FUZZAPI_VERSION: "2"
+ FUZZAPI_VERSION: "3"
FUZZAPI_IMAGE_SUFFIX: ""
FUZZAPI_IMAGE: api-security
@@ -35,9 +35,12 @@ apifuzzer_fuzz:
image: $SECURE_ANALYZERS_PREFIX/$FUZZAPI_IMAGE:$FUZZAPI_VERSION$FUZZAPI_IMAGE_SUFFIX
allow_failure: true
rules:
- - if: $API_FUZZING_DISABLED
+ - if: $API_FUZZING_DISABLED == 'true' || $API_FUZZING_DISABLED == '1'
when: never
- - if: $API_FUZZING_DISABLED_FOR_DEFAULT_BRANCH &&
+ - if: $API_FUZZING_DISABLED_FOR_DEFAULT_BRANCH == 'true' &&
+ $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME
+ when: never
+ - if: $API_FUZZING_DISABLED_FOR_DEFAULT_BRANCH == '1' &&
$CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME
when: never
diff --git a/lib/gitlab/ci/templates/Security/BAS.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/BAS.latest.gitlab-ci.yml
new file mode 100644
index 00000000000..b626a7ca770
--- /dev/null
+++ b/lib/gitlab/ci/templates/Security/BAS.latest.gitlab-ci.yml
@@ -0,0 +1,65 @@
+# To contribute improvements to CI/CD templates, please follow the Development guide at:
+# https://docs.gitlab.com/ee/development/cicd/templates.html
+# This specific template is located at:
+# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/BAS.latest.gitlab-ci.yml
+
+# To use this template, add the following to your .gitlab-ci.yml file:
+#
+# include:
+# template: BAS.latest.gitlab-ci.yml
+#
+# You also need to add a `dast` stage to your `stages:` configuration. A sample configuration for DAST:
+#
+# stages:
+# - build
+# - test
+# - deploy
+# - dast
+#
+# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/breach_and_attack_simulation/index.html#extend-dynamic-application-security-testing-dast
+
+# Include the DAST.latest template if $DAST_VERSION is null because this means a DAST template has not been included already.
+include:
+ - template: Security/DAST.latest.gitlab-ci.yml
+ rules:
+ - if: $DAST_VERSION == null
+
+variables:
+ BAS_CALLBACK_IMAGE_TAG: "latest"
+ BAS_DAST_IMAGE_TAG: "latest"
+ # Setting this variable will affect all Security templates
+ # (SAST, Dependency Scanning, ...)
+ SECURE_ANALYZERS_PREFIX: "$CI_TEMPLATE_REGISTRY_HOST/security-products"
+
+dast_with_bas:
+ extends:
+ - dast
+ - .dast_with_bas
+ rules:
+ # Don't add if the DAST+BAS job is disabled.
+ - if: $DAST_BAS_DISABLED == 'true' || $DAST_BAS_DISABLED == '1'
+ when: never
+ # Add the job to merge request pipelines if there's an open merge request.
+ - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+ # Don't add it to a *branch* pipeline if it's already in a merge request pipeline.
+ - if: $CI_OPEN_MERGE_REQUESTS
+ when: never
+ # If there's no open merge request, add it to a *branch* pipeline instead.
+ - if: $CI_COMMIT_BRANCH
+
+.dast_with_bas:
+ image:
+ name: "$SECURE_ANALYZERS_PREFIX/dast/breach-and-attack-simulation:$BAS_DAST_IMAGE_TAG"
+ variables:
+ DAST_BROWSER_SCAN: "true"
+ DAST_FF_ENABLE_BAS: "true"
+ DAST_FULL_SCAN_ENABLED: "true"
+
+.dast_with_bas_using_services:
+ extends: .dast_with_bas
+ services:
+ - name: "$SECURE_ANALYZERS_PREFIX/callback:$BAS_CALLBACK_IMAGE_TAG"
+ alias: callback
+ variables:
+ DAST_BROWSER_CALLBACK: "Address:http://callback"
+ FF_NETWORK_PER_BUILD: "true"
diff --git a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml
index 89944e347f6..1f11ec8e288 100644
--- a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml
@@ -49,6 +49,6 @@ coverage_fuzzing_unlicensed:
coverage_fuzzing: gl-coverage-fuzzing-report.json
when: always
rules:
- - if: $COVFUZZ_DISABLED
+ - if: $COVFUZZ_DISABLED == 'true' || $COVFUZZ_DISABLED == '1'
when: never
- if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bcoverage_fuzzing\b/
diff --git a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.latest.gitlab-ci.yml
index 4f6ba427058..0cf52468067 100644
--- a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.latest.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.latest.gitlab-ci.yml
@@ -49,7 +49,7 @@ coverage_fuzzing_unlicensed:
coverage_fuzzing: gl-coverage-fuzzing-report.json
when: always
rules:
- - if: $COVFUZZ_DISABLED
+ - if: $COVFUZZ_DISABLED == 'true' || $COVFUZZ_DISABLED == '1'
when: never
# Add the job to merge request pipelines if there's an open merge request.
diff --git a/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml
index 1b33596baa0..ee99d3b4614 100644
--- a/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml
@@ -26,7 +26,7 @@ variables:
# (SAST, Dependency Scanning, ...)
SECURE_ANALYZERS_PREFIX: "$CI_TEMPLATE_REGISTRY_HOST/security-products"
#
- DAST_API_VERSION: "2"
+ DAST_API_VERSION: "3"
DAST_API_IMAGE_SUFFIX: ""
DAST_API_IMAGE: api-security
@@ -35,9 +35,12 @@ dast_api:
image: $SECURE_ANALYZERS_PREFIX/$DAST_API_IMAGE:$DAST_API_VERSION$DAST_API_IMAGE_SUFFIX
allow_failure: true
rules:
- - if: $DAST_API_DISABLED
+ - if: $DAST_API_DISABLED == 'true' || $DAST_API_DISABLED == '1'
when: never
- - if: $DAST_API_DISABLED_FOR_DEFAULT_BRANCH &&
+ - if: $DAST_API_DISABLED_FOR_DEFAULT_BRANCH == 'true' &&
+ $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME
+ when: never
+ - if: $DAST_API_DISABLED_FOR_DEFAULT_BRANCH == '1' &&
$CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME
when: never
- if: $CI_COMMIT_BRANCH &&
diff --git a/lib/gitlab/ci/templates/Security/DAST-API.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-API.latest.gitlab-ci.yml
index a28914d082f..f0b3dc3d2d9 100644
--- a/lib/gitlab/ci/templates/Security/DAST-API.latest.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/DAST-API.latest.gitlab-ci.yml
@@ -26,7 +26,7 @@ variables:
# (SAST, Dependency Scanning, ...)
SECURE_ANALYZERS_PREFIX: "$CI_TEMPLATE_REGISTRY_HOST/security-products"
#
- DAST_API_VERSION: "2"
+ DAST_API_VERSION: "3"
DAST_API_IMAGE_SUFFIX: ""
DAST_API_IMAGE: api-security
@@ -35,9 +35,12 @@ dast_api:
image: $SECURE_ANALYZERS_PREFIX/$DAST_API_IMAGE:$DAST_API_VERSION$DAST_API_IMAGE_SUFFIX
allow_failure: true
rules:
- - if: $DAST_API_DISABLED
+ - if: $DAST_API_DISABLED == 'true' || $DAST_API_DISABLED == '1'
when: never
- - if: $DAST_API_DISABLED_FOR_DEFAULT_BRANCH &&
+ - if: $DAST_API_DISABLED_FOR_DEFAULT_BRANCH == 'true' &&
+ $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME
+ when: never
+ - if: $DAST_API_DISABLED_FOR_DEFAULT_BRANCH == '1' &&
$CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME
when: never
diff --git a/lib/gitlab/ci/templates/Security/DAST-On-Demand-API-Scan.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-On-Demand-API-Scan.gitlab-ci.yml
index 5863da142f0..7b9d16e4192 100644
--- a/lib/gitlab/ci/templates/Security/DAST-On-Demand-API-Scan.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/DAST-On-Demand-API-Scan.gitlab-ci.yml
@@ -14,7 +14,7 @@ stages:
variables:
SECURE_ANALYZERS_PREFIX: "$CI_TEMPLATE_REGISTRY_HOST/security-products"
- DAST_API_VERSION: "2"
+ DAST_API_VERSION: "3"
DAST_API_IMAGE_SUFFIX: ""
DAST_API_IMAGE: api-security
diff --git a/lib/gitlab/ci/templates/Security/DAST-On-Demand-Scan.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-On-Demand-Scan.gitlab-ci.yml
index 733ba4e4954..1ed4cd86e82 100644
--- a/lib/gitlab/ci/templates/Security/DAST-On-Demand-Scan.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/DAST-On-Demand-Scan.gitlab-ci.yml
@@ -13,7 +13,7 @@ stages:
- dast
variables:
- DAST_VERSION: 3
+ DAST_VERSION: 4
# Setting this variable will affect all Security templates
# (SAST, Dependency Scanning, ...)
SECURE_ANALYZERS_PREFIX: "$CI_TEMPLATE_REGISTRY_HOST/security-products"
diff --git a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml
index c43296b5865..792bd7f666b 100644
--- a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml
@@ -22,7 +22,7 @@
# List of available variables: https://docs.gitlab.com/ee/user/application_security/dast/#available-variables
variables:
- DAST_VERSION: 3
+ DAST_VERSION: 4
# Setting this variable will affect all Security templates
# (SAST, Dependency Scanning, ...)
SECURE_ANALYZERS_PREFIX: "$CI_TEMPLATE_REGISTRY_HOST/security-products"
@@ -42,13 +42,23 @@ dast:
reports:
dast: gl-dast-report.json
rules:
- - if: $DAST_DISABLED
+ - if: $DAST_DISABLED == 'true' || $DAST_DISABLED == '1'
when: never
- - if: $DAST_DISABLED_FOR_DEFAULT_BRANCH &&
+ - if: $DAST_DISABLED_FOR_DEFAULT_BRANCH == 'true' &&
$CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME
when: never
+ - if: $DAST_DISABLED_FOR_DEFAULT_BRANCH == '1' &&
+ $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME
+ when: never
+ - if: $CI_DEFAULT_BRANCH != $CI_COMMIT_REF_NAME &&
+ $REVIEW_DISABLED == 'true'
+ when: never
- if: $CI_DEFAULT_BRANCH != $CI_COMMIT_REF_NAME &&
- $REVIEW_DISABLED
+ $REVIEW_DISABLED == '1'
when: never
- if: $CI_COMMIT_BRANCH &&
$GITLAB_FEATURES =~ /\bdast\b/
+ after_script:
+ # Remove any debug.log files because they might contain secrets.
+ - rm -f /zap/wrk/**/debug.log
+ - cp -r /zap/wrk dast_artifacts
diff --git a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml
index 27bcc14bcf5..d1d1c4d7e52 100644
--- a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml
@@ -22,7 +22,7 @@
# List of available variables: https://docs.gitlab.com/ee/user/application_security/dast/#available-variables
variables:
- DAST_VERSION: 3
+ DAST_VERSION: 4
# Setting this variable will affect all Security templates
# (SAST, Dependency Scanning, ...)
SECURE_ANALYZERS_PREFIX: "$CI_TEMPLATE_REGISTRY_HOST/security-products"
@@ -44,13 +44,19 @@ dast:
reports:
dast: gl-dast-report.json
rules:
- - if: $DAST_DISABLED
+ - if: $DAST_DISABLED == 'true' || $DAST_DISABLED == '1'
when: never
- - if: $DAST_DISABLED_FOR_DEFAULT_BRANCH &&
+ - if: $DAST_DISABLED_FOR_DEFAULT_BRANCH == 'true' &&
$CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME
when: never
+ - if: $DAST_DISABLED_FOR_DEFAULT_BRANCH == '1' &&
+ $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME
+ when: never
+ - if: $CI_DEFAULT_BRANCH != $CI_COMMIT_REF_NAME &&
+ $REVIEW_DISABLED == 'true'
+ when: never
- if: $CI_DEFAULT_BRANCH != $CI_COMMIT_REF_NAME &&
- $REVIEW_DISABLED
+ $REVIEW_DISABLED == '1'
when: never
# Add the job to merge request pipelines if there's an open merge request.
diff --git a/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml
index 631f6cecddf..9a43713cc26 100644
--- a/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml
@@ -255,7 +255,7 @@ dast-runner-validation:
api-security:
extends: .download_images
variables:
- SECURE_BINARIES_ANALYZER_VERSION: "2"
+ SECURE_BINARIES_ANALYZER_VERSION: "3"
only:
variables:
- $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" &&
diff --git a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml
index 51bcbd278d5..2661c208665 100644
--- a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml
@@ -24,6 +24,9 @@ validate:
build:
extends: .terraform:build
+ environment:
+ name: $TF_STATE_NAME
+ action: prepare
deploy:
extends: .terraform:deploy
@@ -31,3 +34,4 @@ deploy:
- build
environment:
name: $TF_STATE_NAME
+ action: start
diff --git a/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml
index dd1676f25b6..f16c28e7b60 100644
--- a/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml
@@ -9,7 +9,7 @@
# There is a more opinionated template which we suggest the users to abide,
# which is the lib/gitlab/ci/templates/Terraform.gitlab-ci.yml
image:
- name: "$CI_TEMPLATE_REGISTRY_HOST/gitlab-org/terraform-images/releases/1.1:v0.43.0"
+ name: "$CI_TEMPLATE_REGISTRY_HOST/gitlab-org/terraform-images/releases/1.4:v1.0.0"
variables:
TF_ROOT: ${CI_PROJECT_DIR} # The relative path to the root directory of the Terraform project
@@ -23,24 +23,24 @@ cache:
.terraform:fmt: &terraform_fmt
stage: validate
script:
- - cd "${TF_ROOT}"
- gitlab-terraform fmt
allow_failure: true
.terraform:validate: &terraform_validate
stage: validate
script:
- - cd "${TF_ROOT}"
- gitlab-terraform validate
.terraform:build: &terraform_build
stage: build
script:
- - cd "${TF_ROOT}"
- gitlab-terraform plan
- gitlab-terraform plan-json
resource_group: ${TF_STATE_NAME}
artifacts:
+ # The next line, which disables public access to pipeline artifacts, may not be available everywhere.
+ # See: https://docs.gitlab.com/ee/ci/yaml/#artifactspublic
+ public: false
paths:
- ${TF_ROOT}/plan.cache
reports:
@@ -49,17 +49,16 @@ cache:
.terraform:deploy: &terraform_deploy
stage: deploy
script:
- - cd "${TF_ROOT}"
- gitlab-terraform apply
resource_group: ${TF_STATE_NAME}
rules:
+ - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $TF_AUTO_DEPLOY == "true"
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
when: manual
.terraform:destroy: &terraform_destroy
stage: cleanup
script:
- - cd "${TF_ROOT}"
- gitlab-terraform destroy
resource_group: ${TF_STATE_NAME}
when: manual
diff --git a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml
index bc23a7c2a95..88fe55a44ab 100644
--- a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml
@@ -24,7 +24,6 @@ cache:
.terraform:fmt: &terraform_fmt
stage: validate
script:
- - cd "${TF_ROOT}"
- gitlab-terraform fmt
allow_failure: true
rules:
@@ -36,7 +35,6 @@ cache:
.terraform:validate: &terraform_validate
stage: validate
script:
- - cd "${TF_ROOT}"
- gitlab-terraform validate
rules:
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
@@ -47,11 +45,13 @@ cache:
.terraform:build: &terraform_build
stage: build
script:
- - cd "${TF_ROOT}"
- gitlab-terraform plan
- gitlab-terraform plan-json
resource_group: ${TF_STATE_NAME}
artifacts:
+ # The next line, which disables public access to pipeline artifacts, may not be available everywhere.
+ # See: https://docs.gitlab.com/ee/ci/yaml/#artifactspublic
+ public: false
paths:
- ${TF_ROOT}/plan.cache
reports:
@@ -65,7 +65,6 @@ cache:
.terraform:deploy: &terraform_deploy
stage: deploy
script:
- - cd "${TF_ROOT}"
- gitlab-terraform apply
resource_group: ${TF_STATE_NAME}
rules:
@@ -76,7 +75,6 @@ cache:
.terraform:destroy: &terraform_destroy
stage: cleanup
script:
- - cd "${TF_ROOT}"
- gitlab-terraform destroy
resource_group: ${TF_STATE_NAME}
when: manual
diff --git a/lib/gitlab/ci/templates/Terraform/Module-Base.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Module-Base.gitlab-ci.yml
index e73e6194760..6d5bd7c2172 100644
--- a/lib/gitlab/ci/templates/Terraform/Module-Base.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Terraform/Module-Base.gitlab-ci.yml
@@ -14,7 +14,7 @@ variables:
TERRAFORM_MODULE_DIR: ${CI_PROJECT_DIR} # The relative path to the root directory of the Terraform project.
TERRAFORM_MODULE_NAME: ${CI_PROJECT_NAME} # The name of your Terraform module, must not have any spaces or underscores (will be translated to hyphens).
TERRAFORM_MODULE_SYSTEM: local # The system or provider your Terraform module targets (ex. local, aws, google).
- TERRAFORM_MODULE_VERSION: ${CI_COMMIT_TAG} # The version - it's recommended to follow SemVer for Terraform Module Versioning.
+ TERRAFORM_MODULE_VERSION: ${CI_COMMIT_TAG} # The version - it's recommended to follow SemVer for Terraform Module Versioning.
.terraform-module:fmt:
stage: validate
@@ -29,7 +29,7 @@ variables:
stage: deploy
image: $CI_TEMPLATE_REGISTRY_HOST/gitlab-org/terraform-images/stable:latest
script:
- - TERRAFORM_MODULE_NAME=$(echo "${TERRAFORM_MODULE_NAME}" | tr " _" -) # module-name must not have spaces or underscores, so translate them to hyphens
+ - TERRAFORM_MODULE_NAME=$(echo "${TERRAFORM_MODULE_NAME}" | tr " _" -) # module-name must not have spaces or underscores, so translate them to hyphens
# Builds the Terraform module artifact: a gzipped tar archive with the contents from `$TERRAFORM_MODULE_DIR` without a `.git` directory.
- tar -vczf /tmp/${TERRAFORM_MODULE_NAME}-${TERRAFORM_MODULE_SYSTEM}-${TERRAFORM_MODULE_VERSION}.tgz -C ${TERRAFORM_MODULE_DIR} --exclude=./.git .
# Uploads the Terraform module artifact to the GitLab Terraform Module Registry, see
diff --git a/lib/gitlab/ci/templates/Verify/Load-Performance-Testing.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Load-Performance-Testing.gitlab-ci.yml
index a907915587a..e8d1eb8e1c4 100644
--- a/lib/gitlab/ci/templates/Verify/Load-Performance-Testing.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Verify/Load-Performance-Testing.gitlab-ci.yml
@@ -17,7 +17,7 @@ load_performance:
variables:
K6_IMAGE: grafana/k6
K6_VERSION: 0.41.0
- K6_TEST_FILE: raw.githubusercontent.com/grafana/k6/master/samples/http_get.js
+ K6_TEST_FILE: raw.githubusercontent.com/grafana/k6/master/examples/http_get.js
K6_OPTIONS: ''
K6_DOCKER_OPTIONS: ''
services:
diff --git a/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml b/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml
index 8dfb6c38b55..59b45d865f1 100644
--- a/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml
@@ -80,7 +80,7 @@ deploy_job:
# the artifact files will be copied to:
# P:\Projects\YourApp\Builds\Rev1.0.0.1 - First commit\
- '$commitSubject = git log -1 --pretty=%s'
- - '$deployFolder = $($env:DEPLOY_FOLDER) + "\" + $($env:CI_BUILD_TAG) + " - " + $commitSubject + "\"'
+ - '$deployFolder = $($env:DEPLOY_FOLDER) + "\" + $($env:CI_COMMIT_TAG) + " - " + $commitSubject + "\"'
# xcopy takes care of recursively creating required folders
- 'xcopy /y ".\$env:EXE_RELEASE_FOLDER\YourApp.exe" "$deployFolder"'
diff --git a/lib/gitlab/ci/trace/chunked_io.rb b/lib/gitlab/ci/trace/chunked_io.rb
index 32f64948635..a3f1b472710 100644
--- a/lib/gitlab/ci/trace/chunked_io.rb
+++ b/lib/gitlab/ci/trace/chunked_io.rb
@@ -166,13 +166,6 @@ module Gitlab
end
def destroy!
- # TODO: Remove this logging once we confirmed new live trace architecture is functional.
- # See https://gitlab.com/gitlab-com/gl-infra/infrastructure/issues/4667.
- unless build.has_archived_trace?
- Sidekiq.logger.warn(message: 'The job does not have archived trace but going to be destroyed.',
- job_id: build.id)
- end
-
trace_chunks.fast_destroy_all
@tell = @size = 0
ensure
diff --git a/lib/gitlab/ci/variables/builder.rb b/lib/gitlab/ci/variables/builder.rb
index 89d681c418d..86e54fdfcdf 100644
--- a/lib/gitlab/ci/variables/builder.rb
+++ b/lib/gitlab/ci/variables/builder.rb
@@ -140,11 +140,13 @@ module Gitlab
# Set environment name here so we can access it when evaluating the job's rules
variables.append(key: 'CI_ENVIRONMENT_NAME', value: job.environment) if job.environment
- # legacy variables
- variables.append(key: 'CI_BUILD_NAME', value: job.name)
- variables.append(key: 'CI_BUILD_STAGE', value: job.stage_name)
- variables.append(key: 'CI_BUILD_TRIGGERED', value: 'true') if job.trigger_request
- variables.append(key: 'CI_BUILD_MANUAL', value: 'true') if job.action?
+ if Feature.disabled?(:ci_remove_legacy_predefined_variables, project)
+ # legacy variables
+ variables.append(key: 'CI_BUILD_NAME', value: job.name)
+ variables.append(key: 'CI_BUILD_STAGE', value: job.stage_name)
+ variables.append(key: 'CI_BUILD_TRIGGERED', value: 'true') if job.trigger_request
+ variables.append(key: 'CI_BUILD_MANUAL', value: 'true') if job.action?
+ end
end
end
diff --git a/lib/gitlab/ci/variables/builder/pipeline.rb b/lib/gitlab/ci/variables/builder/pipeline.rb
index 96d6f1673b9..1e7a18d70b0 100644
--- a/lib/gitlab/ci/variables/builder/pipeline.rb
+++ b/lib/gitlab/ci/variables/builder/pipeline.rb
@@ -40,7 +40,7 @@ module Gitlab
attr_reader :pipeline
- def predefined_commit_variables
+ def predefined_commit_variables # rubocop:disable Metrics/AbcSize - Remove this rubocop:disable when FF `ci_remove_legacy_predefined_variables` is removed.
Gitlab::Ci::Variables::Collection.new.tap do |variables|
next variables unless pipeline.sha.present?
@@ -57,7 +57,9 @@ module Gitlab
variables.append(key: 'CI_COMMIT_TIMESTAMP', value: pipeline.git_commit_timestamp.to_s)
variables.append(key: 'CI_COMMIT_AUTHOR', value: pipeline.git_author_full_text.to_s)
- variables.concat(legacy_predefined_commit_variables)
+ if Feature.disabled?(:ci_remove_legacy_predefined_variables, pipeline.project)
+ variables.concat(legacy_predefined_commit_variables)
+ end
end
end
strong_memoize_attr :predefined_commit_variables
@@ -81,7 +83,9 @@ module Gitlab
variables.append(key: 'CI_COMMIT_TAG', value: pipeline.ref)
variables.append(key: 'CI_COMMIT_TAG_MESSAGE', value: git_tag.message)
- variables.concat(legacy_predefined_commit_tag_variables)
+ if Feature.disabled?(:ci_remove_legacy_predefined_variables, pipeline.project)
+ variables.concat(legacy_predefined_commit_tag_variables)
+ end
end
end
strong_memoize_attr :predefined_commit_tag_variables
diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb
index 59acfa80258..c69d9218a66 100644
--- a/lib/gitlab/ci/yaml_processor.rb
+++ b/lib/gitlab/ci/yaml_processor.rb
@@ -94,23 +94,38 @@ module Gitlab
end
def validate_job_needs!(name, job)
- return unless needs = job.dig(:needs, :job)
+ validate_needs_specification!(name, job.dig(:needs, :job))
- validate_duplicate_needs!(name, needs)
+ job[:rules]&.each do |rule|
+ validate_needs_specification!(name, rule.dig(:needs, :job))
+ end
+ end
+
+ def validate_needs_specification!(name, needs)
+ return unless needs
needs.each do |need|
- validate_job_dependency!(name, need[:name], 'need')
+ validate_job_dependency!(name, need[:name], 'need', optional: need[:optional])
end
- end
- def validate_duplicate_needs!(name, needs)
- unless needs.uniq == needs
- error!("#{name} has duplicate entries in the needs section.")
+ duplicated_needs =
+ needs
+ .group_by { |need| need[:name] }
+ .select { |_, items| items.count > 1 }
+ .keys
+
+ unless duplicated_needs.empty?
+ error!("#{name} has the following needs duplicated: #{duplicated_needs.join(', ')}.")
end
end
- def validate_job_dependency!(name, dependency, dependency_type = 'dependency')
+ def validate_job_dependency!(name, dependency, dependency_type = 'dependency', optional: false)
unless @jobs[dependency.to_sym]
+ # Here, we ignore the optional needed job if it is not in the result YAML due to the `include`
+ # rules. In `lib/gitlab/ci/pipeline/seed/build.rb`, we use `optional` again to ignore the
+ # optional needed job in case it is excluded from the pipeline due to the job's rules.
+ return if optional
+
error!("#{name} job: undefined #{dependency_type}: #{dependency}")
end
diff --git a/lib/gitlab/ci/yaml_processor/result.rb b/lib/gitlab/ci/yaml_processor/result.rb
index d867439b10b..6207b595fc6 100644
--- a/lib/gitlab/ci/yaml_processor/result.rb
+++ b/lib/gitlab/ci/yaml_processor/result.rb
@@ -123,7 +123,8 @@ module Gitlab
start_in: job[:start_in],
trigger: job[:trigger],
bridge_needs: job.dig(:needs, :bridge)&.first,
- release: job[:release]
+ release: job[:release],
+ publish: job[:publish]
}.compact }.compact
end
diff --git a/lib/gitlab/color.rb b/lib/gitlab/color.rb
index 7d9280ddba2..c31c8cb5876 100644
--- a/lib/gitlab/color.rb
+++ b/lib/gitlab/color.rb
@@ -9,7 +9,7 @@ module Gitlab
end
module Constants
- DARK = Color.new('#333333')
+ DARK = Color.new('#1F1E24')
LIGHT = Color.new('#FFFFFF')
COLOR_NAME_TO_HEX = {
@@ -194,8 +194,43 @@ module Gitlab
PATTERN.match?(@value)
end
+ # Implementation should match
+ # https://gitlab.com/gitlab-org/gitlab-ui/-/blob/6245128c7256e3d8db164b92e9580c79d47e9183/src/utils/utils.js#L52-55
+ def to_srgb(value)
+ normalized = value / 255.0
+ normalized <= 0.03928 ? normalized / 12.92 : ((normalized + 0.055) / 1.055)**2.4
+ end
+
+ # Implementation should match
+ # https://gitlab.com/gitlab-org/gitlab-ui/-/blob/6245128c7256e3d8db164b92e9580c79d47e9183/src/utils/utils.js#L57-64
+ def relative_luminance(rgb)
+ # WCAG 2.1 formula: https://www.w3.org/TR/WCAG21/#dfn-relative-luminance
+ # -
+ # WCAG 3.0 will use APAC
+ # Using APAC would be the ultimate goal, but was dismissed by engineering as of now
+ # See https://gitlab.com/gitlab-org/gitlab-ui/-/merge_requests/3418#note_1370107090
+ (0.2126 * to_srgb(rgb[0])) + (0.7152 * to_srgb(rgb[1])) + (0.0722 * to_srgb(rgb[2]))
+ end
+
+ # Implementation should match
+ # https://gitlab.com/gitlab-org/gitlab-ui/-/blob/6245128c7256e3d8db164b92e9580c79d47e9183/src/utils/utils.js#L66-91
def light?
- valid? && rgb.sum > 500
+ return false unless valid?
+
+ luminance = relative_luminance(rgb)
+ light_luminance = relative_luminance([255, 255, 255])
+ dark_luminance = relative_luminance([31, 30, 36])
+
+ contrast_light = (light_luminance + 0.05) / (luminance + 0.05)
+ contrast_dark = (luminance + 0.05) / (dark_luminance + 0.05)
+
+ # Using a threshold contrast of 2.4 instead of 3
+ # as this will solve weird color combinations in the mid tones
+ #
+ # Note that this is the negated condition from GitLab UI,
+ # because the GitLab UI implementation returns the text color,
+ # while this defines whether a background color is light
+ !(contrast_light >= 2.4 || contrast_light > contrast_dark)
end
def luminosity
diff --git a/lib/gitlab/color_schemes.rb b/lib/gitlab/color_schemes.rb
index 3884f5f0428..1ba99e23d65 100644
--- a/lib/gitlab/color_schemes.rb
+++ b/lib/gitlab/color_schemes.rb
@@ -46,7 +46,7 @@ module Gitlab
#
# Returns a Scheme
def self.default
- by_id(1)
+ by_id(Gitlab::CurrentSettings.default_syntax_highlighting_theme)
end
# Iterate through each Scheme
diff --git a/lib/gitlab/config/entry/validators.rb b/lib/gitlab/config/entry/validators.rb
index fad2260d818..28cfb6d8fee 100644
--- a/lib/gitlab/config/entry/validators.rb
+++ b/lib/gitlab/config/entry/validators.rb
@@ -356,7 +356,7 @@ module Gitlab
ports_size = value.count
return if ports_size <= 1
- named_ports = value.select { |e| e.is_a?(Hash) }.map { |e| e[:name] }.compact.map(&:downcase)
+ named_ports = value.select { |e| e.is_a?(Hash) }.filter_map { |e| e[:name] }.map(&:downcase)
if ports_size != named_ports.size
record.errors.add(attribute, 'when there is more than one port, a unique name should be added')
diff --git a/lib/gitlab/config/loader/multi_doc_yaml.rb b/lib/gitlab/config/loader/multi_doc_yaml.rb
index 346adc79896..084d32a85bc 100644
--- a/lib/gitlab/config/loader/multi_doc_yaml.rb
+++ b/lib/gitlab/config/loader/multi_doc_yaml.rb
@@ -4,59 +4,50 @@ module Gitlab
module Config
module Loader
class MultiDocYaml
- TooManyDocumentsError = Class.new(Loader::FormatError)
- DataTooLargeError = Class.new(Loader::FormatError)
- NotHashError = Class.new(Loader::FormatError)
+ include Gitlab::Utils::StrongMemoize
- MULTI_DOC_DIVIDER = /^---$/.freeze
+ MULTI_DOC_DIVIDER = /^---\s+/.freeze
- def initialize(config, max_documents:, additional_permitted_classes: [])
+ def initialize(config, max_documents:, additional_permitted_classes: [], reject_empty: false)
+ @config = config
@max_documents = max_documents
- @safe_config = load_config(config, additional_permitted_classes)
+ @additional_permitted_classes = additional_permitted_classes
+ @reject_empty = reject_empty
end
- def load!
- raise TooManyDocumentsError, 'The parsed YAML has too many documents' if too_many_documents?
- raise DataTooLargeError, 'The parsed YAML is too big' if too_big?
- raise NotHashError, 'Invalid configuration format' unless all_hashes?
-
- safe_config.map(&:deep_symbolize_keys)
+ def valid?
+ documents.all?(&:valid?)
end
- private
-
- attr_reader :safe_config, :max_documents
-
- def load_config(config, additional_permitted_classes)
- config.split(MULTI_DOC_DIVIDER).filter_map do |document|
- YAML.safe_load(document,
- permitted_classes: [Symbol, *additional_permitted_classes],
- permitted_symbols: [],
- aliases: true
- )
- end
- rescue Psych::Exception => e
- raise Loader::FormatError, e.message
+ def load_raw!
+ documents.map(&:load_raw!)
end
- def all_hashes?
- safe_config.all?(Hash)
+ def load!
+ documents.map(&:load!)
end
- def too_many_documents?
- safe_config.count > max_documents
- end
+ private
+
+ attr_reader :config, :max_documents, :additional_permitted_classes, :reject_empty
+
+ # Valid YAML files can start with either a leading delimiter or no delimiter.
+ # To avoid counting a leading delimiter towards the document limit,
+ # this method splits the file by one more than the maximum number of permitted documents.
+ # It then discards the first document if it is blank.
+ def documents
+ docs = config
+ .split(MULTI_DOC_DIVIDER, max_documents_including_leading_delimiter)
+ .map { |d| Yaml.new(d, additional_permitted_classes: additional_permitted_classes) }
- def too_big?
- !deep_sizes.all?(&:valid?)
+ docs.shift if docs.first.blank?
+ docs.reject!(&:blank?) if reject_empty
+ docs
end
+ strong_memoize_attr :documents
- def deep_sizes
- safe_config.map do |config|
- Gitlab::Utils::DeepSize.new(config,
- max_size: Gitlab::CurrentSettings.current_application_settings.max_yaml_size_bytes,
- max_depth: Gitlab::CurrentSettings.current_application_settings.max_yaml_depth)
- end
+ def max_documents_including_leading_delimiter
+ max_documents + 1
end
end
end
diff --git a/lib/gitlab/config/loader/yaml.rb b/lib/gitlab/config/loader/yaml.rb
index 7b87b5b8f97..7138663811e 100644
--- a/lib/gitlab/config/loader/yaml.rb
+++ b/lib/gitlab/config/loader/yaml.rb
@@ -1,5 +1,9 @@
# frozen_string_literal: true
+# NOTE: DO NOT use this class for loading GitLab CI configuration files.
+# Instead, use `Gitlab::Ci::Config::Yaml.load!`, which will properly handle
+# CI configuration headers.
+
module Gitlab
module Config
module Loader
@@ -34,6 +38,10 @@ module Gitlab
@symbolized_config ||= load_raw!.deep_symbolize_keys
end
+ def blank?
+ @config.blank?
+ end
+
private
def hash?
diff --git a/lib/gitlab/config_checker/external_database_checker.rb b/lib/gitlab/config_checker/external_database_checker.rb
index ff20833b5be..c95e19940c3 100644
--- a/lib/gitlab/config_checker/external_database_checker.rb
+++ b/lib/gitlab/config_checker/external_database_checker.rb
@@ -21,9 +21,9 @@ module Gitlab
{
type: 'warning',
message: _('Database \'%{database_name}\' is using PostgreSQL %{pg_version_current}, ' \
- 'but PostgreSQL %{pg_version_minimum} is required for this version of GitLab. ' \
- 'Please upgrade your environment to a supported PostgreSQL version, ' \
- 'see %{pg_requirements_url} for details.') % \
+ 'but this version of GitLab requires PostgreSQL %{pg_version_minimum}. ' \
+ 'Please upgrade your environment to a supported PostgreSQL version. ' \
+ 'See %{pg_requirements_url} for details.') % \
{
database_name: database_name,
pg_version_current: database.version,
diff --git a/lib/gitlab/consul/internal.rb b/lib/gitlab/consul/internal.rb
index 1994369dee9..c4feac412e4 100644
--- a/lib/gitlab/consul/internal.rb
+++ b/lib/gitlab/consul/internal.rb
@@ -12,7 +12,7 @@ module Gitlab
class << self
def api_url
Gitlab.config.consul.api_url.to_s.presence if Gitlab.config.consul
- rescue Settingslogic::MissingSetting
+ rescue GitlabSettings::MissingSetting
Gitlab::AppLogger.error('Consul api_url is not present in config/gitlab.yml')
nil
diff --git a/lib/gitlab/content_security_policy/config_loader.rb b/lib/gitlab/content_security_policy/config_loader.rb
index ceca206b084..669c447c09b 100644
--- a/lib/gitlab/content_security_policy/config_loader.rb
+++ b/lib/gitlab/content_security_policy/config_loader.rb
@@ -24,7 +24,7 @@ module Gitlab
'frame_src' => ContentSecurityPolicy::Directives.frame_src,
'img_src' => "'self' data: blob: http: https:",
'manifest_src' => "'self'",
- 'media_src' => "'self' data: http: https:",
+ 'media_src' => "'self' data: blob: http: https:",
'script_src' => ContentSecurityPolicy::Directives.script_src,
'style_src' => ContentSecurityPolicy::Directives.style_src,
'worker_src' => "#{Gitlab::Utils.append_path(Gitlab.config.gitlab.url, 'assets/')} blob: data:",
diff --git a/lib/gitlab/cycle_analytics/stage_summary.rb b/lib/gitlab/cycle_analytics/stage_summary.rb
index fdbf068303f..2276f2fc3c6 100644
--- a/lib/gitlab/cycle_analytics/stage_summary.rb
+++ b/lib/gitlab/cycle_analytics/stage_summary.rb
@@ -45,7 +45,7 @@ module Gitlab
end
def user_has_sufficient_access?
- @project.team.member?(@current_user, Gitlab::Access::REPORTER)
+ @project.member?(@current_user, Gitlab::Access::REPORTER)
end
def serialize(summary_object, with_unit: false)
diff --git a/lib/gitlab/data_builder/deployment.rb b/lib/gitlab/data_builder/deployment.rb
index 7055f64937d..b26f9a61ee1 100644
--- a/lib/gitlab/data_builder/deployment.rb
+++ b/lib/gitlab/data_builder/deployment.rb
@@ -35,6 +35,7 @@ module Gitlab
deployable_id: deployment.deployable_id,
deployable_url: deployable_url,
environment: deployment.environment.name,
+ environment_tier: deployment.environment.tier,
environment_slug: deployment.environment.slug,
environment_external_url: deployment.environment.external_url,
project: deployment.project.hook_attrs,
diff --git a/lib/gitlab/data_builder/pipeline.rb b/lib/gitlab/data_builder/pipeline.rb
index 939eaa377aa..ecb0cc20a64 100644
--- a/lib/gitlab/data_builder/pipeline.rb
+++ b/lib/gitlab/data_builder/pipeline.rb
@@ -45,8 +45,9 @@ module Gitlab
# rubocop: disable CodeReuse/ActiveRecord
def preload_builds(pipeline, association)
- ActiveRecord::Associations::Preloader.new.preload(pipeline,
- {
+ ActiveRecord::Associations::Preloader.new(
+ records: [pipeline],
+ associations: {
association => {
**::Ci::Pipeline::PROJECT_ROUTE_AND_NAMESPACE_ROUTE,
runner: :tags,
@@ -56,7 +57,7 @@ module Gitlab
ci_stage: []
}
}
- )
+ ).call
end
# rubocop: enable CodeReuse/ActiveRecord
diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb
index 756d0afa7e4..4197c87f51f 100644
--- a/lib/gitlab/database.rb
+++ b/lib/gitlab/database.rb
@@ -18,7 +18,7 @@ module Gitlab
# Minimum PostgreSQL version requirement per documentation:
# https://docs.gitlab.com/ee/install/requirements.html#postgresql-requirements
- MINIMUM_POSTGRES_VERSION = 12
+ MINIMUM_POSTGRES_VERSION = 13
# https://www.postgresql.org/docs/9.2/static/datatype-numeric.html
MAX_INT_VALUE = 2147483647
@@ -35,7 +35,7 @@ module Gitlab
MAX_TEXT_SIZE_LIMIT = 1_000_000
# Migrations before this version may have been removed
- MIN_SCHEMA_GITLAB_VERSION = '15.0'
+ MIN_SCHEMA_GITLAB_VERSION = '15.11'
# Schema we store dynamically managed partitions in (e.g. for time partitioning)
DYNAMIC_PARTITIONS_SCHEMA = :gitlab_partitions_dynamic
@@ -51,6 +51,11 @@ module Gitlab
FULLY_QUALIFIED_IDENTIFIER = /^\w+\.\w+$/
+ ## Database Modes
+ MODE_SINGLE_DATABASE = "single-database"
+ MODE_SINGLE_DATABASE_CI_CONNECTION = "single-database-ci-connection"
+ MODE_MULTIPLE_DATABASES = "multiple-databases"
+
def self.database_base_models
@database_base_models ||= {
# Note that we use ActiveRecord::Base here and not ApplicationRecord.
@@ -128,12 +133,29 @@ module Gitlab
Gitlab::Runtime.max_threads + headroom
end
+ # Database configured. Returns true even if the database is shared
def self.has_config?(database_name)
ActiveRecord::Base.configurations
.configs_for(env_name: Rails.env, name: database_name.to_s, include_replicas: true)
.present?
end
+ # Database configured. Returns false if the database is shared
+ def self.has_database?(database_name)
+ db_config = ::Gitlab::Database.database_base_models[database_name]&.connection_db_config
+ db_config.present? && db_config_share_with(db_config).nil?
+ end
+
+ def self.database_mode
+ if !has_config?(CI_DATABASE_NAME)
+ MODE_SINGLE_DATABASE
+ elsif has_database?(CI_DATABASE_NAME)
+ MODE_MULTIPLE_DATABASES
+ else
+ MODE_SINGLE_DATABASE_CI_CONNECTION
+ end
+ end
+
class PgUser < ApplicationRecord
self.table_name = 'pg_user'
self.primary_key = :usename
@@ -171,13 +193,12 @@ module Gitlab
 ███ ███  ██  ██ ██  ██ ██   ████ ██ ██   ████  ██████  
******************************************************************************
- You are using PostgreSQL #{database.version} for the #{name} database, but PostgreSQL >= <%= Gitlab::Database::MINIMUM_POSTGRES_VERSION %>
- is required for this version of GitLab.
+ You are using PostgreSQL #{database.version} for the #{name} database, but this version of GitLab requires PostgreSQL >= <%= Gitlab::Database::MINIMUM_POSTGRES_VERSION %>.
<% if Rails.env.development? || Rails.env.test? %>
If using gitlab-development-kit, please find the relevant steps here:
https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/main/doc/howto/postgresql.md#upgrade-postgresql
<% end %>
- Please upgrade your environment to a supported PostgreSQL version, see
+ Please upgrade your environment to a supported PostgreSQL version. See
https://docs.gitlab.com/ee/install/requirements.html#database for details.
******************************************************************************
EOS
diff --git a/lib/gitlab/database/async_foreign_keys.rb b/lib/gitlab/database/async_constraints.rb
index 115ae9ba2e8..026197c7e40 100644
--- a/lib/gitlab/database/async_foreign_keys.rb
+++ b/lib/gitlab/database/async_constraints.rb
@@ -2,12 +2,12 @@
module Gitlab
module Database
- module AsyncForeignKeys
+ module AsyncConstraints
DEFAULT_ENTRIES_PER_INVOCATION = 2
def self.validate_pending_entries!(how_many: DEFAULT_ENTRIES_PER_INVOCATION)
- PostgresAsyncForeignKeyValidation.ordered.limit(how_many).each do |record|
- ForeignKeyValidator.new(record).perform
+ PostgresAsyncConstraintValidation.ordered.limit(how_many).each do |record|
+ AsyncConstraints::Validators.for(record).perform
end
end
end
diff --git a/lib/gitlab/database/async_foreign_keys/migration_helpers.rb b/lib/gitlab/database/async_constraints/migration_helpers.rb
index b8b9fc6d156..8b4d4ecea04 100644
--- a/lib/gitlab/database/async_foreign_keys/migration_helpers.rb
+++ b/lib/gitlab/database/async_constraints/migration_helpers.rb
@@ -2,7 +2,7 @@
module Gitlab
module Database
- module AsyncForeignKeys
+ module AsyncConstraints
module MigrationHelpers
# Prepares a foreign key for asynchronous validation.
#
@@ -12,7 +12,7 @@ module Gitlab
def prepare_async_foreign_key_validation(table_name, column_name = nil, name: nil)
Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_ddl_mode!
- return unless async_fk_validation_available?
+ return unless async_constraint_validation_available?
fk_name = name || concurrent_foreign_key_name(table_name, column_name)
@@ -20,7 +20,8 @@ module Gitlab
raise missing_schema_object_message(table_name, "foreign key", fk_name)
end
- async_validation = PostgresAsyncForeignKeyValidation
+ async_validation = PostgresAsyncConstraintValidation
+ .foreign_key_type
.find_or_create_by!(name: fk_name, table_name: table_name)
Gitlab::AppLogger.info(
@@ -34,17 +35,20 @@ module Gitlab
def unprepare_async_foreign_key_validation(table_name, column_name = nil, name: nil)
Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_ddl_mode!
- return unless async_fk_validation_available?
+ return unless async_constraint_validation_available?
fk_name = name || concurrent_foreign_key_name(table_name, column_name)
- PostgresAsyncForeignKeyValidation.find_by(name: fk_name).try(&:destroy)
+ PostgresAsyncConstraintValidation
+ .foreign_key_type
+ .find_by(name: fk_name, table_name: table_name)
+ .try(&:destroy!)
end
def prepare_partitioned_async_foreign_key_validation(table_name, column_name = nil, name: nil)
Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_ddl_mode!
- return unless async_fk_validation_available?
+ return unless async_constraint_validation_available?
Gitlab::Database::PostgresPartitionedTable.each_partition(table_name) do |partition|
prepare_async_foreign_key_validation(partition.identifier, column_name, name: name)
@@ -54,17 +58,54 @@ module Gitlab
def unprepare_partitioned_async_foreign_key_validation(table_name, column_name = nil, name: nil)
Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_ddl_mode!
- return unless async_fk_validation_available?
+ return unless async_constraint_validation_available?
Gitlab::Database::PostgresPartitionedTable.each_partition(table_name) do |partition|
unprepare_async_foreign_key_validation(partition.identifier, column_name, name: name)
end
end
+ # Prepares a check constraint for asynchronous validation.
+ #
+ # Stores the constraint information in the postgres_async_foreign_key_validations
+ # table to be executed later.
+ #
+ def prepare_async_check_constraint_validation(table_name, name:)
+ Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_ddl_mode!
+
+ return unless async_constraint_validation_available?
+
+ unless check_constraint_exists?(table_name, name)
+ raise missing_schema_object_message(table_name, "check constraint", name)
+ end
+
+ async_validation = PostgresAsyncConstraintValidation
+ .check_constraint_type
+ .find_or_create_by!(name: name, table_name: table_name)
+
+ Gitlab::AppLogger.info(
+ message: 'Prepared check constraint for async validation',
+ table_name: async_validation.table_name,
+ constraint_name: async_validation.name)
+
+ async_validation
+ end
+
+ def unprepare_async_check_constraint_validation(table_name, name:)
+ Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_ddl_mode!
+
+ return unless async_constraint_validation_available?
+
+ PostgresAsyncConstraintValidation
+ .check_constraint_type
+ .find_by(name: name, table_name: table_name)
+ .try(&:destroy!)
+ end
+
private
- def async_fk_validation_available?
- connection.table_exists?(:postgres_async_foreign_key_validations)
+ def async_constraint_validation_available?
+ PostgresAsyncConstraintValidation.table_available?
end
end
end
diff --git a/lib/gitlab/database/async_constraints/postgres_async_constraint_validation.rb b/lib/gitlab/database/async_constraints/postgres_async_constraint_validation.rb
new file mode 100644
index 00000000000..7fb62948119
--- /dev/null
+++ b/lib/gitlab/database/async_constraints/postgres_async_constraint_validation.rb
@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module AsyncConstraints
+ class PostgresAsyncConstraintValidation < SharedModel
+ include QueueErrorHandlingConcern
+
+ self.table_name = 'postgres_async_foreign_key_validations'
+
+ MAX_IDENTIFIER_LENGTH = Gitlab::Database::MigrationHelpers::MAX_IDENTIFIER_NAME_LENGTH
+ MAX_LAST_ERROR_LENGTH = 10_000
+
+ validates :name, presence: true, uniqueness: { scope: :table_name }, length: { maximum: MAX_IDENTIFIER_LENGTH }
+ validates :table_name, presence: true, length: { maximum: MAX_IDENTIFIER_LENGTH }
+
+ enum constraint_type: { foreign_key: 0, check_constraint: 1 }
+
+ scope :ordered, -> { order(attempts: :asc, id: :asc) }
+ scope :foreign_key_type, -> { constraint_type_exists? ? foreign_key : all }
+ scope :check_constraint_type, -> { check_constraint }
+
+ class << self
+ def table_available?
+ connection.table_exists?(table_name)
+ end
+
+ def constraint_type_exists?
+ connection.column_exists?(table_name, :constraint_type)
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/async_constraints/validators.rb b/lib/gitlab/database/async_constraints/validators.rb
new file mode 100644
index 00000000000..39792a5ee8e
--- /dev/null
+++ b/lib/gitlab/database/async_constraints/validators.rb
@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module AsyncConstraints
+ module Validators
+ MAPPING = {
+ foreign_key: Validators::ForeignKey,
+ check_constraint: Validators::CheckConstraint
+ }.freeze
+
+ def self.for(record)
+ MAPPING
+ .fetch(record.constraint_type.to_sym)
+ .new(record)
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/async_constraints/validators/base.rb b/lib/gitlab/database/async_constraints/validators/base.rb
new file mode 100644
index 00000000000..39a72955d63
--- /dev/null
+++ b/lib/gitlab/database/async_constraints/validators/base.rb
@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module AsyncConstraints
+ module Validators
+ class Base
+ include AsyncDdlExclusiveLeaseGuard
+ extend ::Gitlab::Utils::Override
+
+ TIMEOUT_PER_ACTION = 1.day
+ STATEMENT_TIMEOUT = 12.hours
+
+ def initialize(record)
+ @record = record
+ end
+
+ def perform
+ try_obtain_lease do
+ if constraint_exists?
+ log_info('Starting to validate constraint')
+ validate_constraint_with_error_handling
+ log_info('Finished validating constraint')
+ else
+ log_info(skip_log_message)
+ record.destroy!
+ end
+ end
+ end
+
+ private
+
+ attr_reader :record
+
+ delegate :connection, :name, :table_name, :connection_db_config, to: :record
+
+ def constraint_exists?; end
+
+ def validate_constraint_with_error_handling
+ validate_constraint
+ record.destroy!
+ rescue StandardError => error
+ record.handle_exception!(error)
+
+ Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error)
+ Gitlab::AppLogger.error(message: error.message, **logging_options)
+ end
+
+ def validate_constraint
+ set_statement_timeout do
+ connection.execute(<<~SQL.squish)
+ ALTER TABLE #{connection.quote_table_name(table_name)}
+ VALIDATE CONSTRAINT #{connection.quote_column_name(name)};
+ SQL
+ end
+ end
+
+ def set_statement_timeout
+ connection.execute(format("SET statement_timeout TO '%ds'", STATEMENT_TIMEOUT))
+ yield
+ ensure
+ connection.execute('RESET statement_timeout')
+ end
+
+ def lease_timeout
+ TIMEOUT_PER_ACTION
+ end
+
+ def log_info(message)
+ Gitlab::AppLogger.info(message: message, **logging_options)
+ end
+
+ def skip_log_message
+ "Skipping #{name} validation since it does not exist. " \
+ "The queuing entry will be deleted"
+ end
+
+ def logging_options
+ {
+ class: self.class.name.to_s,
+ connection_name: database_config_name,
+ constraint_name: name,
+ constraint_type: record.constraint_type,
+ table_name: table_name
+ }
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/async_constraints/validators/check_constraint.rb b/lib/gitlab/database/async_constraints/validators/check_constraint.rb
new file mode 100644
index 00000000000..695ecdebc9f
--- /dev/null
+++ b/lib/gitlab/database/async_constraints/validators/check_constraint.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module AsyncConstraints
+ module Validators
+ class CheckConstraint < Base
+ private
+
+ override :constraint_exists?
+ def constraint_exists?
+ Gitlab::Database::Migrations::ConstraintsHelpers
+ .check_constraint_exists?(table_name, name, connection: connection)
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/async_constraints/validators/foreign_key.rb b/lib/gitlab/database/async_constraints/validators/foreign_key.rb
new file mode 100644
index 00000000000..ff6b807c982
--- /dev/null
+++ b/lib/gitlab/database/async_constraints/validators/foreign_key.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module AsyncConstraints
+ module Validators
+ class ForeignKey < Base
+ private
+
+ override :constraint_exists?
+ def constraint_exists?
+ Gitlab::Database::PostgresForeignKey
+ .by_constrained_table_name_or_identifier(table_name)
+ .by_name(name)
+ .exists?
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/async_foreign_keys/foreign_key_validator.rb b/lib/gitlab/database/async_foreign_keys/foreign_key_validator.rb
deleted file mode 100644
index 5958c56a45a..00000000000
--- a/lib/gitlab/database/async_foreign_keys/foreign_key_validator.rb
+++ /dev/null
@@ -1,94 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Database
- module AsyncForeignKeys
- class ForeignKeyValidator
- include AsyncDdlExclusiveLeaseGuard
-
- TIMEOUT_PER_ACTION = 1.day
- STATEMENT_TIMEOUT = 12.hours
-
- def initialize(async_validation)
- @async_validation = async_validation
- end
-
- def perform
- try_obtain_lease do
- if foreign_key_exists?
- log_index_info("Starting to validate foreign key")
- validate_foreign_with_error_handling
- log_index_info("Finished validating foreign key")
- else
- log_index_info(skip_log_message)
- async_validation.destroy!
- end
- end
- end
-
- private
-
- attr_reader :async_validation
-
- delegate :connection, :name, :table_name, :connection_db_config, to: :async_validation
-
- def foreign_key_exists?
- relation = if table_name =~ Gitlab::Database::FULLY_QUALIFIED_IDENTIFIER
- Gitlab::Database::PostgresForeignKey.by_constrained_table_identifier(table_name)
- else
- Gitlab::Database::PostgresForeignKey.by_constrained_table_name(table_name)
- end
-
- relation.by_name(name).exists?
- end
-
- def validate_foreign_with_error_handling
- validate_foreign_key
- async_validation.destroy!
- rescue StandardError => error
- async_validation.handle_exception!(error)
-
- Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error)
- Gitlab::AppLogger.error(message: error.message, **logging_options)
- end
-
- def validate_foreign_key
- set_statement_timeout do
- connection.execute(<<~SQL.squish)
- ALTER TABLE #{connection.quote_table_name(table_name)}
- VALIDATE CONSTRAINT #{connection.quote_column_name(name)};
- SQL
- end
- end
-
- def set_statement_timeout
- connection.execute(format("SET statement_timeout TO '%ds'", STATEMENT_TIMEOUT))
- yield
- ensure
- connection.execute('RESET statement_timeout')
- end
-
- def lease_timeout
- TIMEOUT_PER_ACTION
- end
-
- def log_index_info(message)
- Gitlab::AppLogger.info(message: message, **logging_options)
- end
-
- def skip_log_message
- "Skipping #{name} validation since it does not exist. " \
- "The queuing entry will be deleted"
- end
-
- def logging_options
- {
- fk_name: name,
- table_name: table_name,
- class: self.class.name.to_s
- }
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/database/async_foreign_keys/postgres_async_foreign_key_validation.rb b/lib/gitlab/database/async_foreign_keys/postgres_async_foreign_key_validation.rb
deleted file mode 100644
index de69a3d496f..00000000000
--- a/lib/gitlab/database/async_foreign_keys/postgres_async_foreign_key_validation.rb
+++ /dev/null
@@ -1,21 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Database
- module AsyncForeignKeys
- class PostgresAsyncForeignKeyValidation < SharedModel
- include QueueErrorHandlingConcern
-
- self.table_name = 'postgres_async_foreign_key_validations'
-
- MAX_IDENTIFIER_LENGTH = Gitlab::Database::MigrationHelpers::MAX_IDENTIFIER_NAME_LENGTH
- MAX_LAST_ERROR_LENGTH = 10_000
-
- validates :name, presence: true, uniqueness: true, length: { maximum: MAX_IDENTIFIER_LENGTH }
- validates :table_name, presence: true, length: { maximum: MAX_IDENTIFIER_LENGTH }
-
- scope :ordered, -> { order(attempts: :asc, id: :asc) }
- end
- end
- end
-end
diff --git a/lib/gitlab/database/async_indexes/migration_helpers.rb b/lib/gitlab/database/async_indexes/migration_helpers.rb
index f459c43e0ee..d7128a20a0b 100644
--- a/lib/gitlab/database/async_indexes/migration_helpers.rb
+++ b/lib/gitlab/database/async_indexes/migration_helpers.rb
@@ -77,6 +77,35 @@ module Gitlab
async_index
end
+ def prepare_async_index_from_sql(definition)
+ Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_ddl_mode!
+
+ return unless async_index_creation_available?
+
+ table_name, index_name = extract_table_and_index_names_from_concurrent_index!(definition)
+
+ if index_name_exists?(table_name, index_name)
+ Gitlab::AppLogger.warn(
+ message: 'Index not prepared because it already exists',
+ table_name: table_name,
+ index_name: index_name)
+
+ return
+ end
+
+ async_index = Gitlab::Database::AsyncIndexes::PostgresAsyncIndex.find_or_create_by!(name: index_name) do |rec|
+ rec.table_name = table_name
+ rec.definition = definition
+ end
+
+ Gitlab::AppLogger.info(
+ message: 'Prepared index for async creation',
+ table_name: async_index.table_name,
+ index_name: async_index.name)
+
+ async_index
+ end
+
# Prepares an index for asynchronous destruction.
#
# Stores the index information in the postgres_async_indexes table to be removed later. The
@@ -110,7 +139,30 @@ module Gitlab
end
def async_index_creation_available?
- connection.table_exists?(:postgres_async_indexes)
+ table_exists?(:postgres_async_indexes)
+ end
+
+ private
+
+ delegate :table_exists?, to: :connection, private: true
+
+ def extract_table_and_index_names_from_concurrent_index!(definition)
+ statement = index_statement_from!(definition)
+
+ raise 'Index statement not found!' unless statement
+ raise 'Index must be created concurrently!' unless statement.concurrent
+ raise 'Table does not exist!' unless table_exists?(statement.relation.relname)
+
+ [statement.relation.relname, statement.idxname]
+ end
+
+ # This raises `PgQuery::ParseError` if the given statement
+ # is syntactically incorrect, therefore, validates that the
+ # index definition is correct.
+ def index_statement_from!(definition)
+ parsed_query = PgQuery.parse(definition)
+
+ parsed_query.tree.stmts[0].stmt.index_stmt
end
end
end
diff --git a/lib/gitlab/database/background_migration/batch_optimizer.rb b/lib/gitlab/database/background_migration/batch_optimizer.rb
index c8fdf8281cd..9eb456f6e2e 100644
--- a/lib/gitlab/database/background_migration/batch_optimizer.rb
+++ b/lib/gitlab/database/background_migration/batch_optimizer.rb
@@ -43,11 +43,14 @@ module Gitlab
def optimize!
return unless Feature.enabled?(:optimize_batched_migrations, type: :ops)
- if multiplier = batch_size_multiplier
- max_batch = migration.max_batch_size || MAX_BATCH_SIZE
- migration.batch_size = (migration.batch_size * multiplier).to_i.clamp(MIN_BATCH_SIZE, max_batch)
- migration.save!
- end
+ multiplier = batch_size_multiplier
+ return if multiplier.nil?
+
+ max_batch = migration.max_batch_size || MAX_BATCH_SIZE
+ min_batch = [max_batch, MIN_BATCH_SIZE].min
+
+ migration.batch_size = (migration.batch_size * multiplier).to_i.clamp(min_batch, max_batch)
+ migration.save!
end
private
diff --git a/lib/gitlab/database/background_migration/batched_job.rb b/lib/gitlab/database/background_migration/batched_job.rb
index 6b7ff308c7e..523ab2a9f27 100644
--- a/lib/gitlab/database/background_migration/batched_job.rb
+++ b/lib/gitlab/database/background_migration/batched_job.rb
@@ -4,6 +4,7 @@ module Gitlab
module Database
module BackgroundMigration
SplitAndRetryError = Class.new(StandardError)
+ ReduceSubBatchSizeError = Class.new(StandardError)
class BatchedJob < SharedModel
include EachBatch
@@ -12,6 +13,9 @@ module Gitlab
self.table_name = :batched_background_migration_jobs
MAX_ATTEMPTS = 3
+ MIN_BATCH_SIZE = 1
+ SUB_BATCH_SIZE_REDUCE_FACTOR = 0.75
+ SUB_BATCH_SIZE_THRESHOLD = 65
STUCK_JOBS_TIMEOUT = 1.hour.freeze
TIMEOUT_EXCEPTIONS = [ActiveRecord::StatementTimeout, ActiveRecord::ConnectionTimeoutError,
ActiveRecord::AdapterTimeout, ActiveRecord::LockWaitTimeout,
@@ -59,12 +63,12 @@ module Gitlab
end
after_transition any => :failed do |job, transition|
- error_hash = transition.args.find { |arg| arg[:error].present? }
+ exception, from_sub_batch = job.class.extract_transition_options(transition.args)
- exception = error_hash&.fetch(:error)
+ job.reduce_sub_batch_size! if from_sub_batch && job.can_reduce_sub_batch_size?
job.split_and_retry! if job.can_split?(exception)
- rescue SplitAndRetryError => error
+ rescue SplitAndRetryError, ReduceSubBatchSizeError => error
Gitlab::AppLogger.error(
message: error.message,
batched_job_id: job.id,
@@ -75,9 +79,7 @@ module Gitlab
end
after_transition do |job, transition|
- error_hash = transition.args.find { |arg| arg[:error].present? }
-
- exception = error_hash&.fetch(:error)
+ exception, _ = job.class.extract_transition_options(transition.args)
job.batched_job_transition_logs.create(previous_status: transition.from, next_status: transition.to, exception_class: exception&.class, exception_message: exception&.message)
@@ -100,7 +102,16 @@ module Gitlab
delegate :job_class, :table_name, :column_name, :job_arguments, :job_class_name,
to: :batched_migration, prefix: :migration
- attribute :pause_ms, :integer, default: 100
+ def self.extract_transition_options(args)
+ error_hash = args.find { |arg| arg[:error].present? }
+
+ return [] unless error_hash
+
+ exception = error_hash.fetch(:error)
+ from_sub_batch = error_hash[:from_sub_batch]
+
+ [exception, from_sub_batch]
+ end
def time_efficiency
return unless succeeded?
@@ -113,10 +124,13 @@ module Gitlab
end
def can_split?(exception)
- attempts >= MAX_ATTEMPTS &&
- exception&.class&.in?(TIMEOUT_EXCEPTIONS) &&
- batch_size > sub_batch_size &&
- batch_size > 1
+ return if still_retryable?
+
+ exception.class.in?(TIMEOUT_EXCEPTIONS) && within_batch_size_boundaries?
+ end
+
+ def can_reduce_sub_batch_size?
+ still_retryable? && within_batch_size_boundaries?
end
def split_and_retry!
@@ -165,6 +179,51 @@ module Gitlab
end
end
end
+
+ # It reduces the size of +sub_batch_size+ by 25%
+ def reduce_sub_batch_size!
+ raise ReduceSubBatchSizeError, 'Only sub_batch_size of failed jobs can be reduced' unless failed?
+
+ return if sub_batch_exceeds_threshold?
+
+ with_lock do
+ actual_sub_batch_size = sub_batch_size
+ reduced_sub_batch_size = (sub_batch_size * SUB_BATCH_SIZE_REDUCE_FACTOR).to_i.clamp(1, batch_size)
+
+ update!(sub_batch_size: reduced_sub_batch_size)
+
+ Gitlab::AppLogger.warn(
+ message: 'Sub batch size reduced due to timeout',
+ batched_job_id: id,
+ sub_batch_size: actual_sub_batch_size,
+ reduced_sub_batch_size: reduced_sub_batch_size,
+ attempts: attempts,
+ batched_migration_id: batched_migration.id,
+ job_class_name: migration_job_class_name,
+ job_arguments: migration_job_arguments
+ )
+ end
+ end
+
+ def still_retryable?
+ attempts < MAX_ATTEMPTS
+ end
+
+ def within_batch_size_boundaries?
+ batch_size > MIN_BATCH_SIZE && batch_size > sub_batch_size
+ end
+
+ # It doesn't allow sub-batch size to be reduced lower than the threshold
+ #
+ # @info It will prevent the next iteration to reduce the +sub_batch_size+ lower
+ # than the +SUB_BATCH_SIZE_THRESHOLD+ or 65% of its original size.
+ def sub_batch_exceeds_threshold?
+ initial_sub_batch_size = batched_migration.sub_batch_size
+ reduced_sub_batch_size = (sub_batch_size * SUB_BATCH_SIZE_REDUCE_FACTOR).to_i
+ diff = initial_sub_batch_size - reduced_sub_batch_size
+
+ (1.0 * diff / initial_sub_batch_size * 100).round(2) > SUB_BATCH_SIZE_THRESHOLD
+ end
end
end
end
diff --git a/lib/gitlab/database/background_migration/batched_migration.rb b/lib/gitlab/database/background_migration/batched_migration.rb
index 61a660ad14c..a883996a5c5 100644
--- a/lib/gitlab/database/background_migration/batched_migration.rb
+++ b/lib/gitlab/database/background_migration/batched_migration.rb
@@ -25,6 +25,7 @@ module Gitlab
scope :queue_order, -> { order(id: :asc) }
scope :queued, -> { with_statuses(:active, :paused) }
+ scope :finalizing, -> { with_status(:finalizing) }
scope :ordered_by_created_at_desc, -> { order(created_at: :desc) }
# on_hold_until is a temporary runtime status which puts execution "on hold"
@@ -83,8 +84,6 @@ module Gitlab
end
end
- attribute :pause_ms, :integer, default: 100
-
def self.valid_status
state_machine.states.map(&:name)
end
@@ -221,7 +220,7 @@ module Gitlab
end
def health_context
- HealthStatus::Context.new(connection, [table_name])
+ HealthStatus::Context.new(connection, [table_name], gitlab_schema.to_sym)
end
def hold!(until_time: 10.minutes.from_now)
diff --git a/lib/gitlab/database/background_migration/batched_migration_wrapper.rb b/lib/gitlab/database/background_migration/batched_migration_wrapper.rb
index f1fc3efae9e..8fdaa685ba9 100644
--- a/lib/gitlab/database/background_migration/batched_migration_wrapper.rb
+++ b/lib/gitlab/database/background_migration/batched_migration_wrapper.rb
@@ -15,13 +15,21 @@ module Gitlab
# when starting and finishing execution, and optionally saves batch_metrics
# the migration provides, if any are given.
#
- # The job's batch_metrics are serialized to JSON for storage.
+ # @info The job's batch_metrics are serialized to JSON for storage.
+ #
+ # @info Track exceptions that could happen when processing sub-batches
+ # through +Gitlab::BackgroundMigration::SubBatchTimeoutException+
def perform(batch_tracking_record)
start_tracking_execution(batch_tracking_record)
execute_batch(batch_tracking_record)
batch_tracking_record.succeed!
+ rescue SubBatchTimeoutError => exception
+ caused_by = exception.caused_by
+ batch_tracking_record.failure!(error: caused_by, from_sub_batch: true)
+
+ raise caused_by
rescue Exception => error # rubocop:disable Lint/RescueException
batch_tracking_record.failure!(error: error)
@@ -67,7 +75,8 @@ module Gitlab
sub_batch_size: tracking_record.sub_batch_size,
pause_ms: tracking_record.pause_ms,
job_arguments: tracking_record.migration_job_arguments,
- connection: connection)
+ connection: connection,
+ sub_batch_exception: ::Gitlab::Database::BackgroundMigration::SubBatchTimeoutError)
job_instance.perform
diff --git a/lib/gitlab/database/background_migration/health_status.rb b/lib/gitlab/database/background_migration/health_status.rb
index 506d2996ad5..96905fd0bc5 100644
--- a/lib/gitlab/database/background_migration/health_status.rb
+++ b/lib/gitlab/database/background_migration/health_status.rb
@@ -6,11 +6,12 @@ module Gitlab
module HealthStatus
DEFAULT_INIDICATORS = [
Indicators::AutovacuumActiveOnTable,
- Indicators::WriteAheadLog
+ Indicators::WriteAheadLog,
+ Indicators::PatroniApdex
].freeze
# Rather than passing along the migration, we use a more explicitly defined context
- Context = Struct.new(:connection, :tables)
+ Context = Struct.new(:connection, :tables, :gitlab_schema)
def self.evaluate(migration, indicators = DEFAULT_INIDICATORS)
indicators.map do |indicator|
@@ -30,9 +31,12 @@ module Gitlab
end
def self.log_signal(signal, migration)
- Gitlab::AppLogger.info(
- message: "#{migration} signaled: #{signal}",
- migration_id: migration.id
+ Gitlab::BackgroundMigration::Logger.info(
+ migration_id: migration.id,
+ health_status_indicator: signal.indicator_class.to_s,
+ indicator_signal: signal.short_name,
+ signal_reason: signal.reason,
+ message: "#{migration} signaled: #{signal}"
)
end
end
diff --git a/lib/gitlab/database/background_migration/health_status/indicators/patroni_apdex.rb b/lib/gitlab/database/background_migration/health_status/indicators/patroni_apdex.rb
new file mode 100644
index 00000000000..0dd6dd5c2a4
--- /dev/null
+++ b/lib/gitlab/database/background_migration/health_status/indicators/patroni_apdex.rb
@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module BackgroundMigration
+ module HealthStatus
+ module Indicators
+ class PatroniApdex
+ include Gitlab::Utils::StrongMemoize
+
+ def initialize(context)
+ @context = context
+ end
+
+ def evaluate
+ return Signals::NotAvailable.new(self.class, reason: 'indicator disabled') unless enabled?
+
+ connection_error_message = fetch_connection_error_message
+ return unknown_signal(connection_error_message) if connection_error_message.present?
+
+ apdex_sli = fetch_sli(apdex_sli_query)
+ return unknown_signal('Patroni service apdex can not be calculated') unless apdex_sli.present?
+
+ if apdex_sli.to_f > apdex_slo.to_f
+ Signals::Normal.new(self.class, reason: 'Patroni service apdex is above SLO')
+ else
+ Signals::Stop.new(self.class, reason: 'Patroni service apdex is below SLO')
+ end
+ end
+
+ private
+
+ attr_reader :context
+
+ def enabled?
+ Feature.enabled?(:batched_migrations_health_status_patroni_apdex, type: :ops)
+ end
+
+ def unknown_signal(reason)
+ Signals::Unknown.new(self.class, reason: reason)
+ end
+
+ def fetch_connection_error_message
+ return 'Patroni Apdex Settings not configured' unless database_apdex_settings.present?
+ return 'Prometheus client is not ready' unless client.ready?
+ return 'Apdex SLI query is not configured' unless apdex_sli_query
+ return 'Apdex SLO is not configured' unless apdex_slo
+ end
+
+ def client
+ @client ||= Gitlab::PrometheusClient.new(
+ database_apdex_settings[:prometheus_api_url],
+ allow_local_requests: true,
+ verify: true
+ )
+ end
+
+ def database_apdex_settings
+ @database_apdex_settings ||= Gitlab::CurrentSettings.database_apdex_settings&.with_indifferent_access
+ end
+
+ def apdex_sli_query
+ {
+ gitlab_main: database_apdex_settings[:apdex_sli_query][:main],
+ gitlab_ci: database_apdex_settings[:apdex_sli_query][:ci]
+ }.fetch(context.gitlab_schema.to_sym)
+ end
+ strong_memoize_attr :apdex_sli_query
+
+ def apdex_slo
+ {
+ gitlab_main: database_apdex_settings[:apdex_slo][:main],
+ gitlab_ci: database_apdex_settings[:apdex_slo][:ci]
+ }.fetch(context.gitlab_schema.to_sym)
+ end
+ strong_memoize_attr :apdex_slo
+
+ def fetch_sli(query)
+ response = client.query(query)
+ metric = response&.first || {}
+ value = metric.fetch('value', [])
+
+ Array.wrap(value).second
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/background_migration/health_status/signals.rb b/lib/gitlab/database/background_migration/health_status/signals.rb
index be741a9d91b..534c4330cf2 100644
--- a/lib/gitlab/database/background_migration/health_status/signals.rb
+++ b/lib/gitlab/database/background_migration/health_status/signals.rb
@@ -28,8 +28,6 @@ module Gitlab
end
# :nocov:
- private
-
def short_name
self.class.name.demodulize
end
diff --git a/lib/gitlab/database/background_migration/sub_batch_timeout_error.rb b/lib/gitlab/database/background_migration/sub_batch_timeout_error.rb
new file mode 100644
index 00000000000..815dff11e1f
--- /dev/null
+++ b/lib/gitlab/database/background_migration/sub_batch_timeout_error.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module BackgroundMigration
+ class SubBatchTimeoutError < StandardError
+ def initialize(caused_by)
+ @caused_by = caused_by
+
+ super(caused_by)
+ end
+
+ attr_reader :caused_by
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/background_migration_job.rb b/lib/gitlab/database/background_migration_job.rb
index c0e3016fd3d..5141dd05e4e 100644
--- a/lib/gitlab/database/background_migration_job.rb
+++ b/lib/gitlab/database/background_migration_job.rb
@@ -13,10 +13,6 @@ module Gitlab
for_migration_class(class_name).where('arguments = ?', arguments.to_json) # rubocop:disable Rails/WhereEquals
end
- scope :for_partitioning_migration, -> (class_name, table_name) do
- for_migration_class(class_name).where('arguments ->> 2 = ?', table_name)
- end
-
enum status: {
pending: 0,
succeeded: 1
diff --git a/lib/gitlab/database/batch_count.rb b/lib/gitlab/database/batch_count.rb
index 7a064fb4005..7249cb3e73b 100644
--- a/lib/gitlab/database/batch_count.rb
+++ b/lib/gitlab/database/batch_count.rb
@@ -27,7 +27,7 @@
# batch_sum(User, :sign_in_count)
# batch_sum(Issue.group(:state_id), :weight))
# batch_average(Ci::Pipeline, :duration)
-# batch_average(MergeTrain.group(:status), :duration)
+# batch_average(MergeTrains::Car.group(:status), :duration)
module Gitlab
module Database
module BatchCount
diff --git a/lib/gitlab/database/dynamic_model_helpers.rb b/lib/gitlab/database/dynamic_model_helpers.rb
index 2deb89a0b84..83edf77f37e 100644
--- a/lib/gitlab/database/dynamic_model_helpers.rb
+++ b/lib/gitlab/database/dynamic_model_helpers.rb
@@ -17,7 +17,7 @@ module Gitlab
klass
end
- def each_batch(table_name, connection:, scope: ->(table) { table.all }, of: BATCH_SIZE)
+ def each_batch(table_name, connection:, scope: ->(table) { table.all }, of: BATCH_SIZE, **opts)
if transaction_open?
raise <<~MSG.squish
each_batch should not run inside a transaction, you can disable
@@ -26,13 +26,21 @@ module Gitlab
MSG
end
- scope.call(define_batchable_model(table_name, connection: connection))
- .each_batch(of: of) { |batch| yield batch }
+ opts.select! { |k, _| [:column].include? k }
+
+ batchable_model = define_batchable_model(table_name, connection: connection)
+
+ scope.call(batchable_model)
+ .each_batch(of: of, **opts) { |batch| yield batch, batchable_model }
end
- def each_batch_range(table_name, connection:, scope: ->(table) { table.all }, of: BATCH_SIZE)
- each_batch(table_name, connection: connection, scope: scope, of: of) do |batch|
- yield batch.pick('MIN(id), MAX(id)')
+ def each_batch_range(table_name, connection:, scope: ->(table) { table.all }, of: BATCH_SIZE, **opts)
+ opts.select! { |k, _| [:column].include? k }
+
+ each_batch(table_name, connection: connection, scope: scope, of: of, **opts) do |batch, batchable_model|
+ column = opts.fetch(:column, batchable_model.primary_key)
+
+ yield batch.pick("MIN(#{column}), MAX(#{column})")
end
end
end
diff --git a/lib/gitlab/database/gitlab_schema.rb b/lib/gitlab/database/gitlab_schema.rb
index 38558512b6a..4394c089b22 100644
--- a/lib/gitlab/database/gitlab_schema.rb
+++ b/lib/gitlab/database/gitlab_schema.rb
@@ -19,11 +19,12 @@ module Gitlab
DICTIONARY_PATH = 'db/docs/'
- def self.table_schemas(tables, undefined: true)
- tables.map { |table| table_schema(table, undefined: undefined) }.to_set
+ def self.table_schemas!(tables)
+ tables.map { |table| table_schema!(table) }.to_set
end
- def self.table_schema(name, undefined: true)
+ # rubocop:disable Metrics/CyclomaticComplexity
+ def self.table_schema(name)
schema_name, table_name = name.split('.', 2) # Strip schema name like: `public.`
# Most of names do not have schemas, ensure that this is table
@@ -45,6 +46,11 @@ module Gitlab
return gitlab_schema
end
+ # Partitions that belong to the CI domain
+ if table_name.start_with?('ci_') && gitlab_schema = views_and_tables_to_schema["p_#{table_name}"]
+ return gitlab_schema
+ end
+
# All tables from `information_schema.` are marked as `internal`
return :gitlab_internal if schema_name == 'information_schema'
@@ -52,6 +58,8 @@ module Gitlab
return :gitlab_ci if table_name.start_with?('_test_gitlab_ci_')
+ return :gitlab_embedding if table_name.start_with?('_test_gitlab_embedding_')
+
return :gitlab_geo if table_name.start_with?('_test_gitlab_geo_')
# All tables that start with `_test_` without a following schema are shared and ignored
@@ -60,9 +68,14 @@ module Gitlab
# All `pg_` tables are marked as `internal`
return :gitlab_internal if table_name.start_with?('pg_')
- # When undefined it's best to return a unique name so that we don't incorrectly assume that 2 undefined schemas belong on the same database
- undefined ? :"undefined_#{table_name}" : nil
+ # Sometimes the name of an index can be interpreted as a table's name.
+ # For eg, if we execute "ALTER INDEX my_index..", my_index is interpreted as a table name.
+ # In such cases, we should return the schema of the database table actually
+ # holding that index.
+ index_name = table_name
+ derive_schema_from_index(index_name)
end
+ # rubocop:enable Metrics/CyclomaticComplexity
def self.dictionary_path_globs
[Rails.root.join(DICTIONARY_PATH, '*.yml')]
@@ -85,10 +98,13 @@ module Gitlab
end
def self.table_schema!(name)
- self.table_schema(name, undefined: false) || raise(
+ # rubocop:disable Gitlab/DocUrl
+ self.table_schema(name) || raise(
UnknownSchemaError,
- "Could not find gitlab schema for table #{name}: Any new tables must be added to the database dictionary"
+ "Could not find gitlab schema for table #{name}: Any new or deleted tables must be added to the database dictionary " \
+ "See https://docs.gitlab.com/ee/development/database/database_dictionary.html"
)
+ # rubocop:enable Gitlab/DocUrl
end
def self.deleted_views_and_tables_to_schema
@@ -115,12 +131,31 @@ module Gitlab
@schema_names ||= self.views_and_tables_to_schema.values.to_set
end
+ private_class_method def self.derive_schema_from_index(index_name)
+ index = Gitlab::Database::PostgresIndex.find_by(name: index_name,
+ schema: ApplicationRecord.connection.current_schema)
+
+ return unless index
+
+ table_schema(index.tablename)
+ end
+
private_class_method def self.build_dictionary(path_globs)
Dir.glob(path_globs).each_with_object({}) do |file_path, dic|
data = YAML.load_file(file_path)
key_name = data['table_name'] || data['view_name']
+ # rubocop:disable Gitlab/DocUrl
+ if data['gitlab_schema'].nil?
+ raise(
+ UnknownSchemaError,
+ "#{file_path} must specify a valid gitlab_schema for #{key_name}. " \
+ "See https://docs.gitlab.com/ee/development/database/database_dictionary.html"
+ )
+ end
+ # rubocop:enable Gitlab/DocUrl
+
dic[key_name] = data['gitlab_schema'].to_sym
end
end
diff --git a/lib/gitlab/database/load_balancing/action_cable_callbacks.rb b/lib/gitlab/database/load_balancing/action_cable_callbacks.rb
index 7164976ff73..fab691117ad 100644
--- a/lib/gitlab/database/load_balancing/action_cable_callbacks.rb
+++ b/lib/gitlab/database/load_balancing/action_cable_callbacks.rb
@@ -6,14 +6,10 @@ module Gitlab
module ActionCableCallbacks
def self.install
::ActionCable::Server::Worker.set_callback :work, :around, &wrapper
- ::ActionCable::Channel::Base.set_callback :subscribe, :around, &wrapper
- ::ActionCable::Channel::Base.set_callback :unsubscribe, :around, &wrapper
end
def self.wrapper
lambda do |_, inner|
- ::Gitlab::Database::LoadBalancing::Session.current.use_primary!
-
inner.call
ensure
::Gitlab::Database::LoadBalancing.release_hosts
diff --git a/lib/gitlab/database/load_balancing/connection_proxy.rb b/lib/gitlab/database/load_balancing/connection_proxy.rb
index 622e310ead3..02f14e020c1 100644
--- a/lib/gitlab/database/load_balancing/connection_proxy.rb
+++ b/lib/gitlab/database/load_balancing/connection_proxy.rb
@@ -62,6 +62,13 @@ module Gitlab
end
end
+ def schema_cache(*args, **kwargs, &block)
+ # Ignore primary stickiness for schema_cache queries and always use replicas
+ @load_balancer.read do |connection|
+ connection.public_send(:schema_cache, *args, **kwargs, &block)
+ end
+ end
+
def transaction(*args, **kwargs, &block)
if current_session.fallback_to_replicas_for_ambiguous_queries?
track_read_only_transaction!
diff --git a/lib/gitlab/database/load_balancing/logger.rb b/lib/gitlab/database/load_balancing/logger.rb
index ee67ffcc99c..df6ae47bb84 100644
--- a/lib/gitlab/database/load_balancing/logger.rb
+++ b/lib/gitlab/database/load_balancing/logger.rb
@@ -4,6 +4,8 @@ module Gitlab
module Database
module LoadBalancing
class Logger < ::Gitlab::JsonLogger
+ exclude_context!
+
def self.file_name_noext
'database_load_balancing'
end
diff --git a/lib/gitlab/database/load_balancing/service_discovery.rb b/lib/gitlab/database/load_balancing/service_discovery.rb
index 5059b3b5c93..57a588db8a8 100644
--- a/lib/gitlab/database/load_balancing/service_discovery.rb
+++ b/lib/gitlab/database/load_balancing/service_discovery.rb
@@ -103,6 +103,15 @@ module Gitlab
# Slightly randomize the retry delay so that, in the case of a total
# dns outage, all starting services do not pressure the dns server at the same time.
sleep(rand(RETRY_DELAY_RANGE))
+ rescue Exception => error # rubocop:disable Lint/RescueException
+ # All exceptions are logged to find any pattern and solve https://gitlab.com/gitlab-org/gitlab/-/issues/364370
+ # This will be removed in https://gitlab.com/gitlab-org/gitlab/-/merge_requests/120173
+ Gitlab::Database::LoadBalancing::Logger.error(
+ event: :service_discovery_unexpected_exception,
+ message: "Service discovery encountered an uncaught error: #{error.message}"
+ )
+
+ raise
end
interval
diff --git a/lib/gitlab/database/load_balancing/sidekiq_client_middleware.rb b/lib/gitlab/database/load_balancing/sidekiq_client_middleware.rb
index 619f11ae890..346d951414a 100644
--- a/lib/gitlab/database/load_balancing/sidekiq_client_middleware.rb
+++ b/lib/gitlab/database/load_balancing/sidekiq_client_middleware.rb
@@ -5,13 +5,16 @@ module Gitlab
module LoadBalancing
class SidekiqClientMiddleware
include Gitlab::Utils::StrongMemoize
+ include WalTrackingSender
def call(worker_class, job, _queue, _redis_pool)
# Mailers can't be constantized
worker_class = worker_class.to_s.safe_constantize
+ # ActiveJobs have wrapped class stored in 'wrapped' key
+ resolved_class = job['wrapped'].to_s.safe_constantize || worker_class
- if load_balancing_enabled?(worker_class)
- job['worker_data_consistency'] = worker_class.get_data_consistency
+ if load_balancing_enabled?(resolved_class)
+ job['worker_data_consistency'] = resolved_class.get_data_consistency
set_data_consistency_locations!(job) unless job['wal_locations']
else
job['worker_data_consistency'] = ::WorkerAttributes::DEFAULT_DATA_CONSISTENCY
@@ -24,21 +27,13 @@ module Gitlab
def load_balancing_enabled?(worker_class)
worker_class &&
- worker_class.include?(::ApplicationWorker) &&
+ worker_class.include?(::WorkerAttributes) &&
worker_class.utilizes_load_balancing_capabilities? &&
worker_class.get_data_consistency_feature_flag_enabled?
end
def set_data_consistency_locations!(job)
- locations = {}
-
- ::Gitlab::Database::LoadBalancing.each_load_balancer do |lb|
- if (location = wal_location_for(lb))
- locations[lb.name] = location
- end
- end
-
- job['wal_locations'] = locations
+ job['wal_locations'] = wal_locations_by_db_name
job['wal_location_source'] = wal_location_source
end
@@ -50,17 +45,6 @@ module Gitlab
end
end
- def wal_location_for(load_balancer)
- # When only using the primary there's no need for any WAL queries.
- return if load_balancer.primary_only?
-
- if uses_primary?
- load_balancer.primary_write_location
- else
- load_balancer.host&.database_replica_location || load_balancer.primary_write_location
- end
- end
-
def uses_primary?
::Gitlab::Database::LoadBalancing::Session.current.use_primary?
end
diff --git a/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb b/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb
index f7b8d2514ba..1cb91a5c45b 100644
--- a/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb
+++ b/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb
@@ -4,21 +4,24 @@ module Gitlab
module Database
module LoadBalancing
class SidekiqServerMiddleware
+ include WalTrackingReceiver
+
JobReplicaNotUpToDate = Class.new(::Gitlab::SidekiqMiddleware::RetryError)
- MINIMUM_DELAY_INTERVAL_SECONDS = 0.8
+ REPLICA_WAIT_SLEEP_SECONDS = 0.5
def call(worker, job, _queue)
- worker_class = worker.class
- strategy = select_load_balancing_strategy(worker_class, job)
+ # ActiveJobs have wrapped class stored in 'wrapped' key
+ resolved_class = job['wrapped']&.safe_constantize || worker.class
+ strategy = select_load_balancing_strategy(resolved_class, job)
job['load_balancing_strategy'] = strategy.to_s
if use_primary?(strategy)
::Gitlab::Database::LoadBalancing::Session.current.use_primary!
elsif strategy == :retry
- raise JobReplicaNotUpToDate, "Sidekiq job #{worker_class} JID-#{job['jid']} couldn't use the replica."\
- " Replica was not up to date."
+ raise JobReplicaNotUpToDate, "Sidekiq job #{resolved_class} JID-#{job['jid']} couldn't use the replica."\
+ " Replica was not up to date."
else
# this means we selected an up-to-date replica, but there is nothing to do in this case.
end
@@ -49,7 +52,10 @@ module Gitlab
# Happy case: we can read from a replica.
return replica_strategy(worker_class, job) if databases_in_sync?(wal_locations)
- sleep_if_needed(job)
+ 3.times do
+ sleep REPLICA_WAIT_SLEEP_SECONDS
+ break if databases_in_sync?(wal_locations)
+ end
if databases_in_sync?(wal_locations)
replica_strategy(worker_class, job)
@@ -62,24 +68,18 @@ module Gitlab
end
end
- def sleep_if_needed(job)
- remaining_delay = MINIMUM_DELAY_INTERVAL_SECONDS - (Time.current.to_f - job['created_at'].to_f)
-
- sleep remaining_delay if remaining_delay > 0 && remaining_delay < MINIMUM_DELAY_INTERVAL_SECONDS
- end
-
def get_wal_locations(job)
job['dedup_wal_locations'] || job['wal_locations']
end
def load_balancing_available?(worker_class)
- worker_class.include?(::ApplicationWorker) &&
+ worker_class.include?(::WorkerAttributes) &&
worker_class.utilizes_load_balancing_capabilities? &&
worker_class.get_data_consistency_feature_flag_enabled?
end
def can_retry?(worker_class, job)
- worker_class.get_data_consistency == :delayed && not_yet_retried?(job)
+ worker_class.get_data_consistency == :delayed && not_yet_requeued?(job)
end
def replica_strategy(worker_class, job)
@@ -87,27 +87,14 @@ module Gitlab
end
def retried_before?(worker_class, job)
- worker_class.get_data_consistency == :delayed && !not_yet_retried?(job)
+ worker_class.get_data_consistency == :delayed && !not_yet_requeued?(job)
end
- def not_yet_retried?(job)
+ def not_yet_requeued?(job)
# if `retry_count` is `nil` it indicates that this job was never retried
# the `0` indicates that this is a first retry
job['retry_count'].nil?
end
-
- def databases_in_sync?(wal_locations)
- ::Gitlab::Database::LoadBalancing.each_load_balancer.all? do |lb|
- if (location = wal_locations.with_indifferent_access[lb.name])
- lb.select_up_to_date_host(location)
- else
- # If there's no entry for a load balancer it means the Sidekiq
- # job doesn't care for it. In this case we'll treat the load
- # balancer as being in sync.
- true
- end
- end
- end
end
end
end
diff --git a/lib/gitlab/database/load_balancing/wal_tracking_receiver.rb b/lib/gitlab/database/load_balancing/wal_tracking_receiver.rb
new file mode 100644
index 00000000000..ef9bee42277
--- /dev/null
+++ b/lib/gitlab/database/load_balancing/wal_tracking_receiver.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module LoadBalancing
+ module WalTrackingReceiver
+ # NOTE: If there's no entry for a load balancer or no WAL locations were passed
+ # we assume the sender does not care about LB and we assume nodes are in-sync.
+ def databases_in_sync?(wal_locations)
+ return true unless wal_locations.present?
+
+ ::Gitlab::Database::LoadBalancing.each_load_balancer.all? do |lb|
+ if (location = wal_locations.with_indifferent_access[lb.name])
+ lb.select_up_to_date_host(location)
+ else
+ true
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/load_balancing/wal_tracking_sender.rb b/lib/gitlab/database/load_balancing/wal_tracking_sender.rb
new file mode 100644
index 00000000000..ed0af58d064
--- /dev/null
+++ b/lib/gitlab/database/load_balancing/wal_tracking_sender.rb
@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module LoadBalancing
+ module WalTrackingSender
+ def wal_locations_by_db_name
+ {}.tap do |locations|
+ ::Gitlab::Database::LoadBalancing.each_load_balancer do |lb|
+ if (location = wal_location_for(lb))
+ locations[lb.name] = location
+ end
+ end
+ end
+ end
+
+ def wal_location_for(load_balancer)
+ # When only using the primary there's no need for any WAL queries.
+ return if load_balancer.primary_only?
+
+ if Session.current.use_primary?
+ load_balancer.primary_write_location
+ else
+ load_balancer.host&.database_replica_location || load_balancer.primary_write_location
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/lock_writes_manager.rb b/lib/gitlab/database/lock_writes_manager.rb
index 83884e89d6e..43e71e6bda2 100644
--- a/lib/gitlab/database/lock_writes_manager.rb
+++ b/lib/gitlab/database/lock_writes_manager.rb
@@ -10,6 +10,8 @@ module Gitlab
# See https://www.postgresql.org/message-id/16934.1568989957%40sss.pgh.pa.us
EXPECTED_TRIGGER_RECORD_COUNT = 3
+ # table_name can include schema name as a prefix. For example: 'gitlab_partitions_static.events_03',
+ # otherwise, it will default to current used schema, for example 'public'.
def initialize(table_name:, connection:, database_name:, with_retries: true, logger: nil, dry_run: false)
@table_name = table_name
@connection = connection
@@ -19,7 +21,7 @@ module Gitlab
@with_retries = with_retries
@table_name_without_schema = ActiveRecord::ConnectionAdapters::PostgreSQL::Utils
- .extract_schema_qualified_name(table_name)
+ .extract_schema_qualified_name(table_name.to_s)
.identifier
end
@@ -36,7 +38,7 @@ module Gitlab
def lock_writes
if table_locked_for_writes?
logger&.info "Skipping lock_writes, because #{table_name} is already locked for writes"
- return
+ return result_hash(action: 'skipped')
end
logger&.info "Database: '#{database_name}', Table: '#{table_name}': Lock Writes".color(:yellow)
@@ -48,6 +50,8 @@ module Gitlab
SQL
execute_sql_statement(sql_statement)
+
+ result_hash(action: 'locked')
end
def unlock_writes
@@ -57,6 +61,8 @@ module Gitlab
SQL
execute_sql_statement(sql_statement)
+
+ result_hash(action: 'unlocked')
end
private
@@ -111,6 +117,10 @@ module Gitlab
def write_trigger_name
"gitlab_schema_write_trigger_for_#{table_name_without_schema}"
end
+
+ def result_hash(action:)
+ { action: action, database: database_name, table: table_name, dry_run: dry_run }
+ end
end
end
end
diff --git a/lib/gitlab/database/migration.rb b/lib/gitlab/database/migration.rb
index 4d38920f571..fbb71c1ccfd 100644
--- a/lib/gitlab/database/migration.rb
+++ b/lib/gitlab/database/migration.rb
@@ -53,6 +53,7 @@ module Gitlab
class V2_1 < V2_0 # rubocop:disable Naming/ClassAndModuleCamelCase
include Gitlab::Database::MigrationHelpers::AutomaticLockWritesOnTables
+ include Gitlab::Database::Migrations::RunnerBackoff::MigrationHelpers
end
def self.[](version)
diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb
index 9b041c18da4..291f483e6e4 100644
--- a/lib/gitlab/database/migration_helpers.rb
+++ b/lib/gitlab/database/migration_helpers.rb
@@ -14,7 +14,8 @@ module Gitlab
include DynamicModelHelpers
include RenameTableHelpers
include AsyncIndexes::MigrationHelpers
- include AsyncForeignKeys::MigrationHelpers
+ include AsyncConstraints::MigrationHelpers
+ include WraparoundVacuumHelpers
def define_batchable_model(table_name, connection: self.connection)
super(table_name, connection: connection)
@@ -79,63 +80,6 @@ module Gitlab
end
end
- # @deprecated Use `create_table` in V2 instead
- #
- # Creates a new table, optionally allowing the caller to add check constraints to the table.
- # Aside from that addition, this method should behave identically to Rails' `create_table` method.
- #
- # Example:
- #
- # create_table_with_constraints :some_table do |t|
- # t.integer :thing, null: false
- # t.text :other_thing
- #
- # t.check_constraint :thing_is_not_null, 'thing IS NOT NULL'
- # t.text_limit :other_thing, 255
- # end
- #
- # See Rails' `create_table` for more info on the available arguments.
- def create_table_with_constraints(table_name, **options, &block)
- helper_context = self
-
- with_lock_retries do
- check_constraints = []
-
- create_table(table_name, **options) do |t|
- t.define_singleton_method(:check_constraint) do |name, definition|
- helper_context.send(:validate_check_constraint_name!, name) # rubocop:disable GitlabSecurity/PublicSend
-
- check_constraints << { name: name, definition: definition }
- end
-
- t.define_singleton_method(:text_limit) do |column_name, limit, name: nil|
- # rubocop:disable GitlabSecurity/PublicSend
- name = helper_context.send(:text_limit_name, table_name, column_name, name: name)
- helper_context.send(:validate_check_constraint_name!, name)
- # rubocop:enable GitlabSecurity/PublicSend
-
- column_name = helper_context.quote_column_name(column_name)
- definition = "char_length(#{column_name}) <= #{limit}"
-
- check_constraints << { name: name, definition: definition }
- end
-
- t.instance_eval(&block) unless block.nil?
- end
-
- next if check_constraints.empty?
-
- constraint_clauses = check_constraints.map do |constraint|
- "ADD CONSTRAINT #{quote_table_name(constraint[:name])} CHECK (#{constraint[:definition]})"
- end
-
- execute(<<~SQL)
- ALTER TABLE #{quote_table_name(table_name)}
- #{constraint_clauses.join(",\n")}
- SQL
- end
- end
-
# Creates a new index, concurrently
#
# Example:
@@ -292,23 +236,34 @@ module Gitlab
# order of the ALTER TABLE. This can be useful in situations where the foreign
# key creation could deadlock with another process.
#
- # rubocop: disable Metrics/ParameterLists
- def add_concurrent_foreign_key(source, target, column:, on_delete: :cascade, on_update: nil, target_column: :id, name: nil, validate: true, reverse_lock_order: false)
+ def add_concurrent_foreign_key(source, target, column:, **options)
+ options.reverse_merge!({
+ on_delete: :cascade,
+ on_update: nil,
+ target_column: :id,
+ validate: true,
+ reverse_lock_order: false,
+ allow_partitioned: false,
+ column: column
+ })
+
# Transactions would result in ALTER TABLE locks being held for the
# duration of the transaction, defeating the purpose of this method.
if transaction_open?
raise 'add_concurrent_foreign_key can not be run inside a transaction'
end
- options = {
- column: column,
- on_delete: on_delete,
- on_update: on_update,
- name: name.presence || concurrent_foreign_key_name(source, column),
- primary_key: target_column
- }
+ if !options.delete(:allow_partitioned) && table_partitioned?(source)
+ raise ArgumentError, 'add_concurrent_foreign_key can not be used on a partitioned ' \
+ 'table. Please use add_concurrent_partitioned_foreign_key on the partitioned table ' \
+ 'as we need to create foreign keys on each partition and a FK on the parent table'
+ end
- if foreign_key_exists?(source, target, **options)
+ options[:name] ||= concurrent_foreign_key_name(source, column)
+ options[:primary_key] = options[:target_column]
+ check_options = options.slice(:column, :on_delete, :on_update, :name, :primary_key)
+
+ if foreign_key_exists?(source, target, **check_options)
warning_message = "Foreign key not created because it exists already " \
"(this may be due to an aborted migration or similar): " \
"source: #{source}, target: #{target}, column: #{options[:column]}, "\
@@ -317,23 +272,7 @@ module Gitlab
Gitlab::AppLogger.warn warning_message
else
- # Using NOT VALID allows us to create a key without immediately
- # validating it. This means we keep the ALTER TABLE lock only for a
- # short period of time. The key _is_ enforced for any newly created
- # data.
-
- with_lock_retries do
- execute("LOCK TABLE #{target}, #{source} IN SHARE ROW EXCLUSIVE MODE") if reverse_lock_order
- execute <<-EOF.strip_heredoc
- ALTER TABLE #{source}
- ADD CONSTRAINT #{options[:name]}
- FOREIGN KEY (#{multiple_columns(options[:column])})
- REFERENCES #{target} (#{multiple_columns(target_column)})
- #{on_update_statement(options[:on_update])}
- #{on_delete_statement(options[:on_delete])}
- NOT VALID;
- EOF
- end
+ execute_add_concurrent_foreign_key(source, target, options)
end
# Validate the existing constraint. This can potentially take a very
@@ -345,13 +284,12 @@ module Gitlab
#
# Note this is a no-op in case the constraint is VALID already
- if validate
+ if options[:validate]
disable_statement_timeout do
execute("ALTER TABLE #{source} VALIDATE CONSTRAINT #{options[:name]};")
end
end
end
- # rubocop: enable Metrics/ParameterLists
def validate_foreign_key(source, column, name: nil)
fk_name = name || concurrent_foreign_key_name(source, column)
@@ -379,7 +317,14 @@ module Gitlab
end
end
- fks = Gitlab::Database::PostgresForeignKey.by_constrained_table_name(source)
+ # Since we may be migrating in one go from a previous version without
+ # `constrained_table_name` then we may see that this column exists
+ # (as above) but the schema cache is still outdated for the model.
+ unless Gitlab::Database::PostgresForeignKey.column_names.include?('constrained_table_name')
+ Gitlab::Database::PostgresForeignKey.reset_column_information
+ end
+
+ fks = Gitlab::Database::PostgresForeignKey.by_constrained_table_name_or_identifier(source)
fks = fks.by_referenced_table_name(target) if target
fks = fks.by_name(options[:name]) if options[:name]
@@ -1239,6 +1184,12 @@ into similar problems in the future (e.g. when new tables are created).
end
end
+ def table_partitioned?(table_name)
+ Gitlab::Database::PostgresPartitionedTable
+ .find_by_name_in_current_schema(table_name)
+ .present?
+ end
+
private
def multiple_columns(columns, separator: ', ')
@@ -1354,6 +1305,33 @@ into similar problems in the future (e.g. when new tables are created).
Must end with `_at`}
MESSAGE
end
+
+ def execute_add_concurrent_foreign_key(source, target, options)
+ # Using NOT VALID allows us to create a key without immediately
+ # validating it. This means we keep the ALTER TABLE lock only for a
+ # short period of time. The key _is_ enforced for any newly created
+ # data.
+ not_valid = 'NOT VALID'
+ lock_mode = 'SHARE ROW EXCLUSIVE'
+
+ if table_partitioned?(source)
+ not_valid = ''
+ lock_mode = 'ACCESS EXCLUSIVE'
+ end
+
+ with_lock_retries do
+ execute("LOCK TABLE #{target}, #{source} IN #{lock_mode} MODE") if options[:reverse_lock_order]
+ execute(<<~SQL.squish)
+ ALTER TABLE #{source}
+ ADD CONSTRAINT #{options[:name]}
+ FOREIGN KEY (#{multiple_columns(options[:column])})
+ REFERENCES #{target} (#{multiple_columns(options[:target_column])})
+ #{on_update_statement(options[:on_update])}
+ #{on_delete_statement(options[:on_delete])}
+ #{not_valid};
+ SQL
+ end
+ end
end
end
end
diff --git a/lib/gitlab/database/migration_helpers/automatic_lock_writes_on_tables.rb b/lib/gitlab/database/migration_helpers/automatic_lock_writes_on_tables.rb
index c59139344ea..55c4fd6a7af 100644
--- a/lib/gitlab/database/migration_helpers/automatic_lock_writes_on_tables.rb
+++ b/lib/gitlab/database/migration_helpers/automatic_lock_writes_on_tables.rb
@@ -44,16 +44,7 @@ module Gitlab
# that should be skipped as they will be removed in a future migration.
return false if Gitlab::Database::GitlabSchema.deleted_tables_to_schema[table_name]
- table_schema = Gitlab::Database::GitlabSchema.table_schema(table_name.to_s, undefined: false)
-
- if table_schema.nil?
- error_message = <<~ERROR
- No gitlab_schema is defined for the table #{table_name}. Please consider
- adding it to the database dictionary.
- More info: https://docs.gitlab.com/ee/development/database/database_dictionary.html
- ERROR
- raise error_message
- end
+ table_schema = Gitlab::Database::GitlabSchema.table_schema!(table_name.to_s)
return false unless %i[gitlab_main gitlab_ci].include?(table_schema)
diff --git a/lib/gitlab/database/migration_helpers/convert_to_bigint.rb b/lib/gitlab/database/migration_helpers/convert_to_bigint.rb
new file mode 100644
index 00000000000..63928d7dc09
--- /dev/null
+++ b/lib/gitlab/database/migration_helpers/convert_to_bigint.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module MigrationHelpers
+ module ConvertToBigint
+ # This helper is extracted for the purpose of
+ # https://gitlab.com/gitlab-org/gitlab/-/issues/392815
+ # so that we can test all combinations just once,
+ # and simplify migration tests.
+ #
+ # Once we are done with the PK conversions we can remove this.
+ def com_or_dev_or_test_but_not_jh?
+ return true if Gitlab.dev_or_test_env?
+
+ Gitlab.com? && !Gitlab.jh?
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/migration_helpers/loose_foreign_key_helpers.rb b/lib/gitlab/database/migration_helpers/loose_foreign_key_helpers.rb
index 30601bffd7a..2221aea9f46 100644
--- a/lib/gitlab/database/migration_helpers/loose_foreign_key_helpers.rb
+++ b/lib/gitlab/database/migration_helpers/loose_foreign_key_helpers.rb
@@ -9,11 +9,11 @@ module Gitlab
DELETED_RECORDS_INSERT_FUNCTION_NAME = 'insert_into_loose_foreign_keys_deleted_records'
def track_record_deletions(table)
- execute(<<~SQL)
- CREATE TRIGGER #{record_deletion_trigger_name(table)}
- AFTER DELETE ON #{table} REFERENCING OLD TABLE AS old_table
- FOR EACH STATEMENT
- EXECUTE FUNCTION #{DELETED_RECORDS_INSERT_FUNCTION_NAME}();
+ execute(<<~SQL.squish)
+ CREATE TRIGGER #{record_deletion_trigger_name(table)}
+ AFTER DELETE ON #{table} REFERENCING OLD TABLE AS old_table
+ FOR EACH STATEMENT
+ EXECUTE FUNCTION #{DELETED_RECORDS_INSERT_FUNCTION_NAME}();
SQL
end
@@ -21,6 +21,10 @@ module Gitlab
drop_trigger(table, record_deletion_trigger_name(table))
end
+ def has_loose_foreign_key?(table)
+ trigger_exists?(table, record_deletion_trigger_name(table))
+ end
+
private
def record_deletion_trigger_name(table)
diff --git a/lib/gitlab/database/migration_helpers/v2.rb b/lib/gitlab/database/migration_helpers/v2.rb
index b5b8b58681c..07e22963177 100644
--- a/lib/gitlab/database/migration_helpers/v2.rb
+++ b/lib/gitlab/database/migration_helpers/v2.rb
@@ -5,24 +5,6 @@ module Gitlab
module MigrationHelpers
module V2
include Gitlab::Database::MigrationHelpers
-
- # Superseded by `create_table` override below
- def create_table_with_constraints(*_)
- raise <<~EOM
- #create_table_with_constraints is not supported anymore - use #create_table instead, for example:
-
- create_table :db_guides do |t|
- t.bigint :stars, default: 0, null: false
- t.text :title, limit: 128
- t.text :notes, limit: 1024
-
- t.check_constraint 'stars > 1000', name: 'so_many_stars'
- end
-
- See https://docs.gitlab.com/ee/development/database/strings_and_the_text_data_type.html
- EOM
- end
-
# Creates a new table, optionally allowing the caller to add text limit constraints to the table.
# This method only extends Rails' `create_table` method
#
@@ -192,6 +174,28 @@ module Gitlab
end
end
+ # TRUNCATE is a DDL statement (it drops the table and re-creates it), so we want to run the
+ # migration in DDL mode, but we also don't want to execute it against all schemas because
+ # it will be prevented by the lock_writes trigger.
+ #
+ # For example,
+ # a `gitlab_main` table on `:gitlab_main` database will be truncated,
+ # and a `gitlab_main` table on `:gitlab_ci` database will be skipped.
+ #
+ # Note Rails already has a truncate_tables, see
+ # https://github.com/rails/rails/blob/6-1-stable/activerecord/lib/active_record/connection_adapters/abstract/database_statements.rb#L193
+ def truncate_tables!(*table_names, connection: self.connection)
+ table_schemas = Gitlab::Database::GitlabSchema.table_schemas!(table_names)
+
+ raise ArgumentError, "`table_names` must resolve to only one `gitlab_schema`" if table_schemas.size != 1
+
+ return unless Gitlab::Database.gitlab_schemas_for_connection(connection).include?(table_schemas.first)
+
+ quoted_tables = table_names.map { |table_name| quote_table_name(table_name) }.join(', ')
+
+ execute("TRUNCATE TABLE #{quoted_tables}")
+ end
+
private
def setup_renamed_column(calling_operation, table, old_column, new_column, type, batch_column_name)
diff --git a/lib/gitlab/database/migration_helpers/wraparound_vacuum_helpers.rb b/lib/gitlab/database/migration_helpers/wraparound_vacuum_helpers.rb
new file mode 100644
index 00000000000..2a9d37452bd
--- /dev/null
+++ b/lib/gitlab/database/migration_helpers/wraparound_vacuum_helpers.rb
@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module MigrationHelpers
+ module WraparoundVacuumHelpers
+ class WraparoundCheck
+ WraparoundError = Class.new(StandardError)
+
+ def initialize(table_name, migration:)
+ @migration = migration
+ @table_name = table_name
+
+ validate_table_existence!
+ end
+
+ def execute
+ return if disabled?
+ return unless wraparound_vacuum.present?
+
+ log "Autovacuum with wraparound prevention mode is running on `#{table_name}`", title: true
+ log "This process prevents the migration from acquiring the necessary locks"
+ log "Query: `#{wraparound_vacuum[:query]}`"
+ log "Current duration: #{wraparound_vacuum[:duration].inspect}"
+ log "You can wait until it completes or if absolutely necessary interrupt it, " \
+ "but be aware that a new process will kick in immediately, so multiple interruptions " \
+ "might be required to time it right with the locks retry mechanism"
+ end
+
+ private
+
+ attr_reader :table_name
+
+ delegate :say, :connection, to: :@migration
+
+ def wraparound_vacuum
+ @wraparound_vacuum ||= transform_wraparound_vacuum
+ end
+
+ def transform_wraparound_vacuum
+ result = raw_wraparound_vacuum
+ values = Array.wrap(result.cast_values.first)
+
+ result.columns.zip(values).to_h.with_indifferent_access.compact
+ end
+
+ def raw_wraparound_vacuum
+ connection.select_all(<<~SQL.squish)
+ SELECT age(clock_timestamp(), query_start) as duration, query
+ FROM postgres_pg_stat_activity_autovacuum()
+ WHERE query ILIKE '%VACUUM%' || #{quoted_table_name} || '%(to prevent wraparound)'
+ LIMIT 1
+ SQL
+ end
+
+ def validate_table_existence!
+ return if connection.table_exists?(table_name)
+
+ raise WraparoundError, "Table #{table_name} does not exist"
+ end
+
+ def quoted_table_name
+ connection.quote(table_name)
+ end
+
+ def disabled?
+ return true unless wraparound_check_allowed?
+
+ Gitlab::Utils.to_boolean(ENV['GITLAB_MIGRATIONS_DISABLE_WRAPAROUND_CHECK'])
+ end
+
+ def wraparound_check_allowed?
+ Gitlab.com? || Gitlab.dev_or_test_env?
+ end
+
+ def log(text, title: false)
+ say text, !title
+ end
+ end
+
+ def check_if_wraparound_in_progress(table_name)
+ WraparoundCheck.new(table_name, migration: self).execute
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/migrations/batched_background_migration_helpers.rb b/lib/gitlab/database/migrations/batched_background_migration_helpers.rb
index e958ce2aba4..cb2a98b553f 100644
--- a/lib/gitlab/database/migrations/batched_background_migration_helpers.rb
+++ b/lib/gitlab/database/migrations/batched_background_migration_helpers.rb
@@ -12,6 +12,7 @@ module Gitlab
# For now, these migrations are not considered ready for general use, for more information see the tracking epic:
# https://gitlab.com/groups/gitlab-org/-/epics/6751
module BatchedBackgroundMigrationHelpers
+ NonExistentMigrationError = Class.new(StandardError)
BATCH_SIZE = 1_000 # Number of rows to process per job
SUB_BATCH_SIZE = 100 # Number of rows to process per sub-batch
BATCH_CLASS_NAME = 'PrimaryKeyBatchingStrategy' # Default batch class for batched migrations
@@ -200,6 +201,12 @@ module Gitlab
def ensure_batched_background_migration_is_finished(job_class_name:, table_name:, column_name:, job_arguments:, finalize: true)
Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_dml_mode!
+ if transaction_open?
+ raise 'The `ensure_batched_background_migration_is_finished` cannot be run inside a transaction. ' \
+ 'You can disable transactions by calling `disable_ddl_transaction!` in the body of ' \
+ 'your migration class.'
+ end
+
Gitlab::Database::BackgroundMigration::BatchedMigration.reset_column_information
migration = Gitlab::Database::BackgroundMigration::BatchedMigration.find_for_configuration(
Gitlab::Database.gitlab_schemas_for_connection(connection),
@@ -213,6 +220,10 @@ module Gitlab
job_arguments: job_arguments
}
+ if ENV['DBLAB_ENVIRONMENT'] && migration.nil?
+ raise NonExistentMigrationError, 'called ensure_batched_background_migration_is_finished with non-existent migration name'
+ end
+
return Gitlab::AppLogger.warn "Could not find batched background migration for the given configuration: #{configuration}" if migration.nil?
return if migration.finished?
diff --git a/lib/gitlab/database/migrations/constraints_helpers.rb b/lib/gitlab/database/migrations/constraints_helpers.rb
index 7b849e3137a..5aafc9f1444 100644
--- a/lib/gitlab/database/migrations/constraints_helpers.rb
+++ b/lib/gitlab/database/migrations/constraints_helpers.rb
@@ -10,6 +10,27 @@ module Gitlab
# https://www.postgresql.org/docs/current/sql-syntax-lexical.html#SQL-SYNTAX-IDENTIFIERS
MAX_IDENTIFIER_NAME_LENGTH = 63
+ def self.check_constraint_exists?(table, constraint_name, connection:)
+ # Constraint names are unique per table in Postgres, not per schema
+ # Two tables can have constraints with the same name, so we filter by
+ # the table name in addition to using the constraint_name
+
+ check_sql = <<~SQL
+ SELECT COUNT(*)
+ FROM pg_catalog.pg_constraint con
+ INNER JOIN pg_catalog.pg_class rel
+ ON rel.oid = con.conrelid
+ INNER JOIN pg_catalog.pg_namespace nsp
+ ON nsp.oid = con.connamespace
+ WHERE con.contype = 'c'
+ AND con.conname = #{connection.quote(constraint_name)}
+ AND nsp.nspname = #{connection.quote(connection.current_schema)}
+ AND rel.relname = #{connection.quote(table)}
+ SQL
+
+ connection.select_value(check_sql.squish) > 0
+ end
+
# Returns the name for a check constraint
#
# type:
@@ -29,24 +50,7 @@ module Gitlab
end
def check_constraint_exists?(table, constraint_name)
- # Constraint names are unique per table in Postgres, not per schema
- # Two tables can have constraints with the same name, so we filter by
- # the table name in addition to using the constraint_name
-
- check_sql = <<~SQL
- SELECT COUNT(*)
- FROM pg_catalog.pg_constraint con
- INNER JOIN pg_catalog.pg_class rel
- ON rel.oid = con.conrelid
- INNER JOIN pg_catalog.pg_namespace nsp
- ON nsp.oid = con.connamespace
- WHERE con.contype = 'c'
- AND con.conname = #{connection.quote(constraint_name)}
- AND nsp.nspname = #{connection.quote(current_schema)}
- AND rel.relname = #{connection.quote(table)}
- SQL
-
- connection.select_value(check_sql) > 0
+ ConstraintsHelpers.check_constraint_exists?(table, constraint_name, connection: connection)
end
# Adds a check constraint to a table
diff --git a/lib/gitlab/database/migrations/instrumentation.rb b/lib/gitlab/database/migrations/instrumentation.rb
index 8c479d7eda2..5f87bc6bbe2 100644
--- a/lib/gitlab/database/migrations/instrumentation.rb
+++ b/lib/gitlab/database/migrations/instrumentation.rb
@@ -29,6 +29,8 @@ module Gitlab
observation.success = true
observation
+ rescue StandardError => error
+ observation.error_message = error.message
ensure
observation.walltime = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start
diff --git a/lib/gitlab/database/migrations/observation.rb b/lib/gitlab/database/migrations/observation.rb
index cd048beac96..1be62496806 100644
--- a/lib/gitlab/database/migrations/observation.rb
+++ b/lib/gitlab/database/migrations/observation.rb
@@ -4,7 +4,7 @@
module Gitlab
module Database
module Migrations
- Observation = Struct.new(:version, :name, :walltime, :success, :total_database_size_change,
+ Observation = Struct.new(:version, :name, :walltime, :success, :total_database_size_change, :error_message,
:meta, :query_statistics, keyword_init: true) do
def to_json(...)
as_json.except('meta').to_json(...)
diff --git a/lib/gitlab/database/migrations/pg_backend_pid.rb b/lib/gitlab/database/migrations/pg_backend_pid.rb
new file mode 100644
index 00000000000..b59eb55cc6e
--- /dev/null
+++ b/lib/gitlab/database/migrations/pg_backend_pid.rb
@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module Migrations
+ module PgBackendPid
+ module MigratorPgBackendPid
+ extend ::Gitlab::Utils::Override
+
+ override :with_advisory_lock_connection
+ def with_advisory_lock_connection
+ super do |conn|
+ Gitlab::Database::Migrations::PgBackendPid.say(conn)
+
+ yield(conn)
+
+ Gitlab::Database::Migrations::PgBackendPid.say(conn)
+ end
+ end
+ end
+
+ def self.patch!
+ ActiveRecord::Migrator.prepend(MigratorPgBackendPid)
+ end
+
+ def self.say(conn)
+ return unless ActiveRecord::Migration.verbose
+
+ pg_backend_pid = conn.select_value('SELECT pg_backend_pid()')
+ db_name = Gitlab::Database.db_config_name(conn)
+
+ # rubocop:disable Rails/Output
+ puts "#{db_name}: == [advisory_lock_connection] " \
+ "object_id: #{conn.object_id}, pg_backend_pid: #{pg_backend_pid}"
+ # rubocop:enable Rails/Output
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/migrations/runner_backoff/active_record_mixin.rb b/lib/gitlab/database/migrations/runner_backoff/active_record_mixin.rb
new file mode 100644
index 00000000000..b5348f4b4e6
--- /dev/null
+++ b/lib/gitlab/database/migrations/runner_backoff/active_record_mixin.rb
@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module Migrations
+ module RunnerBackoff
+ module ActiveRecordMixin
+ module ActiveRecordMigrationProxyRunnerBackoff
+ # Regular AR migrations don't have this,
+ # only ones inheriting from Gitlab::Database::Migration have
+ def enable_runner_backoff?
+ !!migration.try(:enable_runner_backoff?)
+ end
+ end
+
+ module ActiveRecordMigratorRunnerBackoff
+ def execute_migration_in_transaction(migration)
+ if migration.enable_runner_backoff?
+ RunnerBackoff::Communicator.execute_with_lock(migration) { super }
+ else
+ super
+ end
+ end
+ end
+
+ def self.patch!
+ ActiveRecord::MigrationProxy.prepend(ActiveRecordMigrationProxyRunnerBackoff)
+ ActiveRecord::Migrator.prepend(ActiveRecordMigratorRunnerBackoff)
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/migrations/runner_backoff/communicator.rb b/lib/gitlab/database/migrations/runner_backoff/communicator.rb
new file mode 100644
index 00000000000..874dfc56832
--- /dev/null
+++ b/lib/gitlab/database/migrations/runner_backoff/communicator.rb
@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module Migrations
+ module RunnerBackoff
+ class Communicator
+ EXPIRY = 1.minute
+ KEY = 'gitlab/database/migration/runner/backoff'
+
+ def self.execute_with_lock(migration, &block)
+ new(migration).execute_with_lock(&block)
+ end
+
+ def self.backoff_runner?
+ return false if ::Feature.disabled?(:runner_migrations_backoff, type: :ops)
+
+ Gitlab::ExclusiveLease.new(KEY, timeout: EXPIRY).exists?
+ end
+
+ def initialize(migration, logger: Gitlab::AppLogger)
+ @migration = migration
+ @logger = logger
+ end
+
+ def execute_with_lock
+ log(message: 'Executing migration with Runner backoff')
+
+ set_lock
+ yield if block_given?
+ ensure
+ remove_lock
+ end
+
+ private
+
+ attr_reader :logger, :migration
+
+ def set_lock
+ raise 'Could not set backoff key' unless exclusive_lease.try_obtain
+
+ log(message: 'Runner backoff key is set')
+ end
+
+ def remove_lock
+ exclusive_lease.cancel
+
+ log(message: 'Runner backoff key was removed')
+ end
+
+ def exclusive_lease
+ @exclusive_lease ||= Gitlab::ExclusiveLease.new(KEY, timeout: EXPIRY)
+ end
+
+ def log(params)
+ logger.info(log_params.merge(params))
+ end
+
+ def log_params
+ { class: migration.name }
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/migrations/runner_backoff/migration_helpers.rb b/lib/gitlab/database/migrations/runner_backoff/migration_helpers.rb
new file mode 100644
index 00000000000..b9e564d85e6
--- /dev/null
+++ b/lib/gitlab/database/migrations/runner_backoff/migration_helpers.rb
@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module Migrations
+ module RunnerBackoff
+ module MigrationHelpers
+ extend ActiveSupport::Concern
+
+ class_methods do
+ def enable_runner_backoff!
+ @enable_runner_backoff = true # rubocop:disable Gitlab/ModuleWithInstanceVariables
+ end
+
+ def enable_runner_backoff?
+ !!@enable_runner_backoff
+ end
+ end
+
+ delegate :enable_runner_backoff?, to: :class
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/migrations/sidekiq_helpers.rb b/lib/gitlab/database/migrations/sidekiq_helpers.rb
index c536b33bbdf..6ab14c0fbe6 100644
--- a/lib/gitlab/database/migrations/sidekiq_helpers.rb
+++ b/lib/gitlab/database/migrations/sidekiq_helpers.rb
@@ -25,9 +25,14 @@ module Gitlab
times_in_a_row: DEFAULT_TIMES_IN_A_ROW,
max_attempts: DEFAULT_MAX_ATTEMPTS
)
-
kwargs = { times_in_a_row: times_in_a_row, max_attempts: max_attempts }
+ if transaction_open?
+ raise 'sidekiq_remove_jobs can not be run inside a transaction, ' \
+ 'you can disable transactions by calling disable_ddl_transaction! ' \
+ 'in the body of your migration class'
+ end
+
job_klasses_queues = job_klasses
.select { |job_klass| job_klass.to_s.safe_constantize.present? }
.map { |job_klass| job_klass.safe_constantize.queue }
diff --git a/lib/gitlab/database/migrations/test_batched_background_runner.rb b/lib/gitlab/database/migrations/test_batched_background_runner.rb
index 01fdba22c19..af853c933ba 100644
--- a/lib/gitlab/database/migrations/test_batched_background_runner.rb
+++ b/lib/gitlab/database/migrations/test_batched_background_runner.rb
@@ -27,7 +27,7 @@ module Gitlab
table_max_value = define_batchable_model(migration.table_name, connection: connection)
.maximum(migration.column_name)
- largest_batch_start = table_max_value - migration.batch_size
+ largest_batch_start = [table_max_value - migration.batch_size, smallest_batch_start].max
# variance is the portion of the batch range that we shrink between variance * 0 and variance * 1
# to pick actual batches to sample.
diff --git a/lib/gitlab/database/obsolete_ignored_columns.rb b/lib/gitlab/database/obsolete_ignored_columns.rb
index 2b88ab12380..c172d15301a 100644
--- a/lib/gitlab/database/obsolete_ignored_columns.rb
+++ b/lib/gitlab/database/obsolete_ignored_columns.rb
@@ -2,15 +2,15 @@
module Gitlab
module Database
- # Checks which `ignored_columns` can be safely removed by scanning
- # the current schema for all `ApplicationRecord` descendants.
+ # Checks which `ignored_columns` definitions can be safely removed by
+ # scanning the current schema for all `ApplicationRecord` descendants.
class ObsoleteIgnoredColumns
def initialize(base = ApplicationRecord)
@base = base
end
def execute
- @base.descendants.map do |klass|
+ @base.descendants.filter_map do |klass|
next if klass.abstract_class?
safe_to_remove = ignored_columns_safe_to_remove_for(klass)
diff --git a/lib/gitlab/database/partitioning.rb b/lib/gitlab/database/partitioning.rb
index 6314aff9914..7222f148b3f 100644
--- a/lib/gitlab/database/partitioning.rb
+++ b/lib/gitlab/database/partitioning.rb
@@ -27,6 +27,8 @@ module Gitlab
end
def sync_partitions(models_to_sync = registered_for_sync, only_on: nil)
+ return unless Feature.enabled?(:partition_manager_sync_partitions, type: :ops)
+
Gitlab::AppLogger.info(message: 'Syncing dynamic postgres partitions')
Gitlab::Database::EachDatabase.each_model_connection(models_to_sync, only_on: only_on) do |model|
@@ -37,8 +39,9 @@ module Gitlab
models_to_sync.each do |model|
next if model < ::Gitlab::Database::SharedModel && !(model < TableWithoutModel)
+ model_connection_name = model.connection_db_config.name
Gitlab::Database::EachDatabase.each_database_connection do |connection, connection_name|
- if connection_name != model.connection_db_config.name
+ if connection_name != model_connection_name
PartitionManager.new(model, connection: connection).sync_partitions
end
end
diff --git a/lib/gitlab/database/partitioning/ci_sliding_list_strategy.rb b/lib/gitlab/database/partitioning/ci_sliding_list_strategy.rb
new file mode 100644
index 00000000000..69a69091b5c
--- /dev/null
+++ b/lib/gitlab/database/partitioning/ci_sliding_list_strategy.rb
@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module Partitioning
+ class CiSlidingListStrategy < SlidingListStrategy
+ def initial_partition
+ partition_for(100)
+ end
+
+ def next_partition
+ partition_for(active_partition.value + 1)
+ end
+
+ def validate_and_fix; end
+
+ def after_adding_partitions; end
+
+ def extra_partitions
+ []
+ end
+
+ private
+
+ def ensure_partitioning_column_ignored_or_readonly!; end
+
+ def partition_for(value)
+ SingleNumericListPartition.new(table_name, value, partition_name: partition_name(value))
+ end
+
+ def partition_name(value)
+ [
+ table_name.to_s.delete_prefix('p_'),
+ value
+ ].join('_')
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/partitioning/convert_table_to_first_list_partition.rb b/lib/gitlab/database/partitioning/convert_table_to_first_list_partition.rb
deleted file mode 100644
index 58447481e60..00000000000
--- a/lib/gitlab/database/partitioning/convert_table_to_first_list_partition.rb
+++ /dev/null
@@ -1,268 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Database
- module Partitioning
- class ConvertTableToFirstListPartition
- UnableToPartition = Class.new(StandardError)
-
- SQL_STATEMENT_SEPARATOR = ";\n\n"
-
- attr_reader :partitioning_column, :table_name, :parent_table_name, :zero_partition_value
-
- def initialize(
- migration_context:, table_name:, parent_table_name:, partitioning_column:,
- zero_partition_value:, lock_tables: [])
-
- @migration_context = migration_context
- @connection = migration_context.connection
- @table_name = table_name
- @parent_table_name = parent_table_name
- @partitioning_column = partitioning_column
- @zero_partition_value = zero_partition_value
- @lock_tables = Array.wrap(lock_tables)
- end
-
- def prepare_for_partitioning
- assert_existing_constraints_partitionable
-
- add_partitioning_check_constraint
- end
-
- def revert_preparation_for_partitioning
- migration_context.remove_check_constraint(table_name, partitioning_constraint.name)
- end
-
- def partition
- assert_existing_constraints_partitionable
- assert_partitioning_constraint_present
- create_parent_table
- attach_foreign_keys_to_parent
-
- lock_args = {
- raise_on_exhaustion: true,
- timing_configuration: lock_timing_configuration
- }
-
- migration_context.with_lock_retries(**lock_args) do
- migration_context.execute(sql_to_convert_table)
- end
- end
-
- def revert_partitioning
- migration_context.with_lock_retries(raise_on_exhaustion: true) do
- migration_context.execute(<<~SQL)
- ALTER TABLE #{connection.quote_table_name(parent_table_name)}
- DETACH PARTITION #{connection.quote_table_name(table_name)};
- SQL
-
- alter_sequences_sql = alter_sequence_statements(old_table: parent_table_name, new_table: table_name)
- .join(SQL_STATEMENT_SEPARATOR)
-
- migration_context.execute(alter_sequences_sql)
-
- # This takes locks for all the foreign keys that the parent table had.
- # However, those same locks were taken while detaching the partition, and we can't avoid that.
- # If we dropped the foreign key before detaching the partition to avoid this locking,
- # the drop would cascade to the child partitions and drop their foreign keys as well
- migration_context.drop_table(parent_table_name)
- end
-
- add_partitioning_check_constraint
- end
-
- private
-
- attr_reader :connection, :migration_context
-
- delegate :quote_table_name, :quote_column_name, to: :connection
-
- def sql_to_convert_table
- # The critical statement here is the attach_table_to_parent statement.
- # The following statements could be run in a later transaction,
- # but they acquire the same locks so it's much faster to incude them
- # here.
- [
- lock_tables_statement,
- attach_table_to_parent_statement,
- alter_sequence_statements(old_table: table_name, new_table: parent_table_name),
- remove_constraint_statement
- ].flatten.join(SQL_STATEMENT_SEPARATOR)
- end
-
- def table_identifier
- "#{connection.current_schema}.#{table_name}"
- end
-
- def assert_existing_constraints_partitionable
- violating_constraints = Gitlab::Database::PostgresConstraint
- .by_table_identifier(table_identifier)
- .primary_or_unique_constraints
- .not_including_column(partitioning_column)
- .to_a
-
- return if violating_constraints.empty?
-
- violation_messages = violating_constraints.map { |c| "#{c.name} on (#{c.column_names.join(', ')})" }
-
- raise UnableToPartition, <<~MSG
- Constraints on #{table_name} are incompatible with partitioning on #{partitioning_column}
-
- All primary key and unique constraints must include the partitioning column.
- Violations:
- #{violation_messages.join("\n")}
- MSG
- end
-
- def partitioning_constraint
- constraints_on_column = Gitlab::Database::PostgresConstraint
- .by_table_identifier(table_identifier)
- .check_constraints
- .valid
- .including_column(partitioning_column)
-
- constraints_on_column.to_a.find do |constraint|
- constraint.definition == "CHECK ((#{partitioning_column} = #{zero_partition_value}))"
- end
- end
-
- def assert_partitioning_constraint_present
- return if partitioning_constraint
-
- raise UnableToPartition, <<~MSG
- Table #{table_name} is not ready for partitioning.
- Before partitioning, a check constraint must enforce that (#{partitioning_column} = #{zero_partition_value})
- MSG
- end
-
- def add_partitioning_check_constraint
- return if partitioning_constraint.present?
-
- check_body = "#{partitioning_column} = #{connection.quote(zero_partition_value)}"
- # Any constraint name would work. The constraint is found based on its definition before partitioning
- migration_context.add_check_constraint(table_name, check_body, 'partitioning_constraint')
-
- raise UnableToPartition, 'Error adding partitioning constraint' unless partitioning_constraint.present?
- end
-
- def create_parent_table
- migration_context.execute(<<~SQL)
- CREATE TABLE IF NOT EXISTS #{quote_table_name(parent_table_name)} (
- LIKE #{quote_table_name(table_name)} INCLUDING ALL
- ) PARTITION BY LIST(#{quote_column_name(partitioning_column)})
- SQL
- end
-
- def attach_foreign_keys_to_parent
- migration_context.foreign_keys(table_name).each do |fk|
- # At this point no other connection knows about the parent table.
- # Thus the only contended lock in the following transaction is on fk.to_table.
- # So a deadlock is impossible.
-
- # If we're rerunning this migration after a failure to acquire a lock, the foreign key might already exist.
- # Don't try to recreate it in that case
- if migration_context.foreign_keys(parent_table_name)
- .any? { |p_fk| p_fk.options[:name] == fk.options[:name] }
- next
- end
-
- migration_context.with_lock_retries(raise_on_exhaustion: true) do
- migration_context.add_foreign_key(parent_table_name, fk.to_table, **fk.options)
- end
- end
- end
-
- def lock_tables_statement
- return if @lock_tables.empty?
-
- table_names = @lock_tables.map { |name| quote_table_name(name) }.join(', ')
-
- <<~SQL
- LOCK #{table_names} IN ACCESS EXCLUSIVE MODE
- SQL
- end
-
- def attach_table_to_parent_statement
- <<~SQL
- ALTER TABLE #{quote_table_name(parent_table_name)}
- ATTACH PARTITION #{table_name}
- FOR VALUES IN (#{zero_partition_value})
- SQL
- end
-
- def alter_sequence_statements(old_table:, new_table:)
- sequences_owned_by(old_table).map do |seq_info|
- seq_name, column_name = seq_info.values_at(:name, :column_name)
-
- statement_parts = []
-
- # If a different user owns the old table, the conversion process will fail to reassign the sequence
- # ownership to the new parent table (as it will be owned by the current user).
- # Force the old table to be owned by the current user in that case.
- unless current_user_owns_table?(old_table)
- statement_parts << set_current_user_owns_table_statement(old_table)
- end
-
- statement_parts << <<~SQL.chomp
- ALTER SEQUENCE #{quote_table_name(seq_name)} OWNED BY #{quote_table_name(new_table)}.#{quote_column_name(column_name)}
- SQL
-
- statement_parts.join(SQL_STATEMENT_SEPARATOR)
- end
- end
-
- def remove_constraint_statement
- <<~SQL
- ALTER TABLE #{quote_table_name(parent_table_name)}
- DROP CONSTRAINT #{quote_table_name(partitioning_constraint.name)}
- SQL
- end
-
- # TODO: https://gitlab.com/gitlab-org/gitlab/-/issues/373887
- def sequences_owned_by(table_name)
- sequence_data = connection.exec_query(<<~SQL, nil, [table_name])
- SELECT seq_pg_class.relname AS seq_name,
- dep_pg_class.relname AS table_name,
- pg_attribute.attname AS col_name
- FROM pg_class seq_pg_class
- INNER JOIN pg_depend ON seq_pg_class.oid = pg_depend.objid
- INNER JOIN pg_class dep_pg_class ON pg_depend.refobjid = dep_pg_class.oid
- INNER JOIN pg_attribute ON dep_pg_class.oid = pg_attribute.attrelid
- AND pg_depend.refobjsubid = pg_attribute.attnum
- WHERE seq_pg_class.relkind = 'S'
- AND dep_pg_class.relname = $1
- SQL
-
- sequence_data.map do |seq_info|
- name, column_name = seq_info.values_at('seq_name', 'col_name')
- { name: name, column_name: column_name }
- end
- end
-
- def table_owner(table_name)
- connection.select_value(<<~SQL, nil, [table_name])
- SELECT tableowner FROM pg_tables WHERE tablename = $1
- SQL
- end
-
- def current_user_owns_table?(table_name)
- current_user = connection.select_value('select current_user')
- table_owner(table_name) == current_user
- end
-
- def set_current_user_owns_table_statement(table_name)
- <<~SQL.chomp
- ALTER TABLE #{connection.quote_table_name(table_name)} OWNER TO CURRENT_USER
- SQL
- end
-
- def lock_timing_configuration
- iterations = Gitlab::Database::WithLockRetries::DEFAULT_TIMING_CONFIGURATION
- aggressive_iterations = Array.new(5) { [10.seconds, 1.minute] }
-
- iterations + aggressive_iterations
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/database/partitioning/list/convert_table.rb b/lib/gitlab/database/partitioning/list/convert_table.rb
new file mode 100644
index 00000000000..d4fb150d956
--- /dev/null
+++ b/lib/gitlab/database/partitioning/list/convert_table.rb
@@ -0,0 +1,317 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module Partitioning
+ module List
+ class ConvertTable
+ UnableToPartition = Class.new(StandardError)
+
+ SQL_STATEMENT_SEPARATOR = ";\n\n"
+
+ PARTITIONING_CONSTRAINT_NAME = 'partitioning_constraint'
+
+ attr_reader :partitioning_column, :table_name, :parent_table_name, :zero_partition_value,
+ :locking_configuration
+
+ def initialize(
+ migration_context:, table_name:, parent_table_name:, partitioning_column:,
+ zero_partition_value:, lock_tables: [])
+
+ @migration_context = migration_context
+ @connection = migration_context.connection
+ @table_name = table_name
+ @parent_table_name = parent_table_name
+ @partitioning_column = partitioning_column
+ @zero_partition_value = zero_partition_value
+ @locking_configuration = LockingConfiguration.new(migration_context, table_locking_order: lock_tables)
+ end
+
+ def prepare_for_partitioning(async: false)
+ assert_existing_constraints_partitionable
+
+ add_partitioning_check_constraint(async: async)
+ end
+
+ def revert_preparation_for_partitioning
+ migration_context.remove_check_constraint(table_name, partitioning_constraint.name)
+ end
+
+ def partition
+ assert_existing_constraints_partitionable
+ assert_partitioning_constraint_present
+
+ create_parent_table
+ attach_foreign_keys_to_parent
+ locking_sql = locking_configuration.locking_statement_for(tables_that_will_lock_during_partitioning)
+
+ locking_configuration.with_lock_retries do
+ # Loose FKs trigger will exclusively lock the table and it might
+ # not follow the locking order needed to attach the partition.
+ migration_context.execute(locking_sql) if locking_sql.present?
+
+ redefine_loose_foreign_key_triggers do
+ migration_context.execute(sql_to_convert_table)
+ end
+ end
+ end
+
+ def revert_partitioning
+ migration_context.with_lock_retries(raise_on_exhaustion: true) do
+ migration_context.execute(<<~SQL)
+ ALTER TABLE #{connection.quote_table_name(parent_table_name)}
+ DETACH PARTITION #{connection.quote_table_name(table_name)};
+ SQL
+
+ alter_sequences_sql = alter_sequence_statements(old_table: parent_table_name, new_table: table_name)
+ .join(SQL_STATEMENT_SEPARATOR)
+
+ migration_context.execute(alter_sequences_sql)
+
+ # This takes locks for all the foreign keys that the parent table had.
+ # However, those same locks were taken while detaching the partition, and we can't avoid that.
+ # If we dropped the foreign key before detaching the partition to avoid this locking,
+ # the drop would cascade to the child partitions and drop their foreign keys as well
+ migration_context.drop_table(parent_table_name)
+ end
+
+ add_partitioning_check_constraint
+ end
+
+ private
+
+ attr_reader :connection, :migration_context
+
+ delegate :quote_table_name, :quote_column_name, :current_schema, to: :connection
+
+ def sql_to_convert_table
+ # The critical statement here is the attach_table_to_parent statement.
+ # The following statements could be run in a later transaction,
+ # but they acquire the same locks so it's much faster to include them
+ # here.
+ [
+ attach_table_to_parent_statement,
+ alter_sequence_statements(old_table: table_name, new_table: parent_table_name),
+ remove_constraint_statement
+ ].flatten.join(SQL_STATEMENT_SEPARATOR)
+ end
+
+ def table_identifier
+ "#{current_schema}.#{table_name}"
+ end
+
+ def assert_existing_constraints_partitionable
+ violating_constraints = Gitlab::Database::PostgresConstraint
+ .by_table_identifier(table_identifier)
+ .primary_or_unique_constraints
+ .not_including_column(partitioning_column)
+ .to_a
+
+ return if violating_constraints.empty?
+
+ violation_messages = violating_constraints.map { |c| "#{c.name} on (#{c.column_names.join(', ')})" }
+
+ raise UnableToPartition, <<~MSG
+ Constraints on #{table_name} are incompatible with partitioning on #{partitioning_column}
+
+ All primary key and unique constraints must include the partitioning column.
+ Violations:
+ #{violation_messages.join("\n")}
+ MSG
+ end
+
+ def partitioning_constraint
+ constraints_on_column = Gitlab::Database::PostgresConstraint
+ .by_table_identifier(table_identifier)
+ .check_constraints
+ .including_column(partitioning_column)
+
+ check_body = "CHECK ((#{partitioning_column} = #{zero_partition_value}))"
+
+ constraints_on_column.find do |constraint|
+ constraint.definition.start_with?(check_body)
+ end
+ end
+
+ def assert_partitioning_constraint_present
+ return if partitioning_constraint&.constraint_valid?
+
+ raise UnableToPartition, <<~MSG
+ Table #{table_name} is not ready for partitioning.
+ Before partitioning, a check constraint must enforce that (#{partitioning_column} = #{zero_partition_value})
+ MSG
+ end
+
+ def add_partitioning_check_constraint(async: false)
+ return validate_partitioning_constraint_synchronously if partitioning_constraint.present?
+
+ check_body = "#{partitioning_column} = #{connection.quote(zero_partition_value)}"
+ # Any constraint name would work. The constraint is found based on its definition before partitioning
+ migration_context.add_check_constraint(
+ table_name, check_body, PARTITIONING_CONSTRAINT_NAME,
+ validate: !async
+ )
+
+ if async
+ migration_context.prepare_async_check_constraint_validation(
+ table_name, name: PARTITIONING_CONSTRAINT_NAME
+ )
+ end
+
+ return if partitioning_constraint.present?
+
+ raise UnableToPartition, <<~MSG
+ Error adding partitioning constraint `#{PARTITIONING_CONSTRAINT_NAME}` for `#{table_name}`
+ MSG
+ end
+
+ def validate_partitioning_constraint_synchronously
+ if partitioning_constraint.constraint_valid?
+ return Gitlab::AppLogger.info <<~MSG
+ Nothing to do, the partitioning constraint exists and is valid for `#{table_name}`
+ MSG
+ end
+
+ # Async validations are executed only on .com, we need to validate synchronously for self-managed
+ migration_context.validate_check_constraint(table_name, partitioning_constraint.name)
+ return if partitioning_constraint.constraint_valid?
+
+ raise UnableToPartition, <<~MSG
+ Error validating partitioning constraint `#{partitioning_constraint.name}` for `#{table_name}`
+ MSG
+ end
+
+ def create_parent_table
+ migration_context.execute(<<~SQL)
+ CREATE TABLE IF NOT EXISTS #{quote_table_name(parent_table_name)} (
+ LIKE #{quote_table_name(table_name)} INCLUDING ALL
+ ) PARTITION BY LIST(#{quote_column_name(partitioning_column)})
+ SQL
+ end
+
+ def attach_foreign_keys_to_parent
+ migration_context.foreign_keys(table_name).each do |fk|
+ # At this point no other connection knows about the parent table.
+ # Thus the only contended lock in the following transaction is on fk.to_table.
+ # So a deadlock is impossible.
+
+ # If we're rerunning this migration after a failure to acquire a lock, the foreign key might already exist
+ # Don't try to recreate it in that case
+ if migration_context.foreign_keys(parent_table_name)
+ .any? { |p_fk| p_fk.options[:name] == fk.options[:name] }
+ next
+ end
+
+ migration_context.with_lock_retries(raise_on_exhaustion: true) do
+ migration_context.add_foreign_key(parent_table_name, fk.to_table, **fk.options)
+ end
+ end
+ end
+
+ def attach_table_to_parent_statement
+ <<~SQL
+ ALTER TABLE #{quote_table_name(parent_table_name)}
+ ATTACH PARTITION #{table_name}
+ FOR VALUES IN (#{zero_partition_value})
+ SQL
+ end
+
+ def alter_sequence_statements(old_table:, new_table:)
+ sequences_owned_by(old_table).map do |seq_info|
+ seq_name, column_name = seq_info.values_at(:name, :column_name)
+
+ statement_parts = []
+
+ # If a different user owns the old table, the conversion process will fail to reassign the sequence
+ # ownership to the new parent table (as it will be owned by the current user).
+ # Force the old table to be owned by the current user in that case.
+ unless current_user_owns_table?(old_table)
+ statement_parts << set_current_user_owns_table_statement(old_table)
+ end
+
+ statement_parts << <<~SQL.chomp
+ ALTER SEQUENCE #{quote_table_name(seq_name)} OWNED BY #{quote_table_name(new_table)}.#{quote_column_name(column_name)}
+ SQL
+
+ statement_parts.join(SQL_STATEMENT_SEPARATOR)
+ end
+ end
+
+ def remove_constraint_statement
+ <<~SQL
+ ALTER TABLE #{quote_table_name(parent_table_name)}
+ DROP CONSTRAINT #{quote_table_name(partitioning_constraint.name)}
+ SQL
+ end
+
+ # TODO: https://gitlab.com/gitlab-org/gitlab/-/issues/373887
+ def sequences_owned_by(table_name)
+ sequence_data = connection.exec_query(<<~SQL, nil, [table_name])
+ SELECT seq_pg_class.relname AS seq_name,
+ dep_pg_class.relname AS table_name,
+ pg_attribute.attname AS col_name
+ FROM pg_class seq_pg_class
+ INNER JOIN pg_depend ON seq_pg_class.oid = pg_depend.objid
+ INNER JOIN pg_class dep_pg_class ON pg_depend.refobjid = dep_pg_class.oid
+ INNER JOIN pg_attribute ON dep_pg_class.oid = pg_attribute.attrelid
+ AND pg_depend.refobjsubid = pg_attribute.attnum
+ WHERE seq_pg_class.relkind = 'S'
+ AND dep_pg_class.relname = $1
+ SQL
+
+ sequence_data.map do |seq_info|
+ name, column_name = seq_info.values_at('seq_name', 'col_name')
+ { name: name, column_name: column_name }
+ end
+ end
+
+ def table_owner(table_name)
+ connection.select_value(<<~SQL, nil, [table_name])
+ SELECT tableowner FROM pg_tables WHERE tablename = $1
+ SQL
+ end
+
+ def current_user_owns_table?(table_name)
+ current_user = connection.select_value('select current_user')
+ table_owner(table_name) == current_user
+ end
+
+ def set_current_user_owns_table_statement(table_name)
+ <<~SQL.chomp
+ ALTER TABLE #{connection.quote_table_name(table_name)} OWNER TO CURRENT_USER
+ SQL
+ end
+
+ def table_name_for_identifier(table_identifier)
+ /^\w+\.(\w+)*$/.match(table_identifier)[1]
+ end
+
+ def redefine_loose_foreign_key_triggers
+ if migration_context.has_loose_foreign_key?(table_name)
+ migration_context.untrack_record_deletions(table_name)
+
+ yield if block_given?
+
+ migration_context.track_record_deletions(parent_table_name)
+ migration_context.track_record_deletions(table_name)
+ elsif block_given?
+ yield
+ end
+ end
+
+ def tables_that_will_lock_during_partitioning
+ # Locks are taken against the table + all tables that reference it by foreign key
+ # postgres_foreign_keys.referenced_table_name gives the table name that we need here directly, but that
+ # column did not exist yet during the migration 20221021145820_create_routing_table_for_builds_metadata_v2
+ # To ensure compatibility with that migration if it is run with this code, use referenced_table_identifier
+ # here.
+ referenced_tables = Gitlab::Database::PostgresForeignKey
+ .by_constrained_table_identifier(table_identifier)
+ .map { |fk| table_name_for_identifier(fk.referenced_table_identifier) }
+ referenced_tables + [table_name]
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/partitioning/list/locking_configuration.rb b/lib/gitlab/database/partitioning/list/locking_configuration.rb
new file mode 100644
index 00000000000..02d20383de4
--- /dev/null
+++ b/lib/gitlab/database/partitioning/list/locking_configuration.rb
@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module Partitioning
+ module List
+ class LockingConfiguration
+ attr_reader :migration_context
+
+ def initialize(migration_context, table_locking_order:)
+ @migration_context = migration_context
+ @table_locking_order = table_locking_order.map(&:to_s)
+ assert_table_names_unqualified!(@table_locking_order)
+ end
+
+ def locking_statement_for(tables)
+ tables_to_lock = locking_order_for(tables)
+
+ return if tables_to_lock.empty?
+
+ table_names = tables_to_lock.map { |name| migration_context.quote_table_name(name) }.join(', ')
+
+ <<~SQL
+ LOCK #{table_names} IN ACCESS EXCLUSIVE MODE
+ SQL
+ end
+
+ # Sorts and subsets `tables` to the tables that were explicitly requested for locking
+ # in the order that that locking was requested.
+ def locking_order_for(tables)
+ tables = Array.wrap(tables)
+ assert_table_names_unqualified!(tables)
+
+ @table_locking_order.intersection(tables.map(&:to_s))
+ end
+
+ def lock_timing_configuration
+ iterations = Gitlab::Database::WithLockRetries::DEFAULT_TIMING_CONFIGURATION
+ aggressive_iterations = Array.new(5) { [10.seconds, 1.minute] }
+
+ iterations + aggressive_iterations
+ end
+
+ def with_lock_retries(&block)
+ lock_args = {
+ raise_on_exhaustion: true,
+ timing_configuration: lock_timing_configuration
+ }
+
+ migration_context.with_lock_retries(**lock_args, &block)
+ end
+
+ private
+
+ def assert_table_names_unqualified!(table_names)
+ tables = Array.wrap(table_names).select { |name| name.to_s.include?('.') }
+ return if tables.empty?
+
+ raise ArgumentError, "All table names must be unqualified, but #{tables.join(', ')} include schema"
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/partitioning/partition_manager.rb b/lib/gitlab/database/partitioning/partition_manager.rb
index 55ca9ff8645..124fae582d3 100644
--- a/lib/gitlab/database/partitioning/partition_manager.rb
+++ b/lib/gitlab/database/partitioning/partition_manager.rb
@@ -34,6 +34,8 @@ module Gitlab
create(partitions_to_create) unless partitions_to_create.empty?
detach(partitions_to_detach) unless partitions_to_detach.empty?
end
+ rescue ArgumentError => e
+ Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e)
rescue StandardError => e
Gitlab::AppLogger.error(
message: "Failed to create / detach partition(s)",
diff --git a/lib/gitlab/database/partitioning/replace_table.rb b/lib/gitlab/database/partitioning/replace_table.rb
index 21a175a660d..60871a64845 100644
--- a/lib/gitlab/database/partitioning/replace_table.rb
+++ b/lib/gitlab/database/partitioning/replace_table.rb
@@ -70,7 +70,7 @@ module Gitlab
def rename_partitions(statements, old_table_name, new_table_name)
Gitlab::Database::PostgresPartition.for_parent_table(old_table_name).each do |partition|
- new_partition_name = partition.name.sub(/#{old_table_name}/, new_table_name)
+ new_partition_name = partition.name.sub(/#{old_table_name}/, new_table_name.to_s)
old_primary_key = default_primary_key(partition.name)
new_primary_key = default_primary_key(new_partition_name)
diff --git a/lib/gitlab/database/partitioning_migration_helpers/backfill_partitioned_table.rb b/lib/gitlab/database/partitioning_migration_helpers/backfill_partitioned_table.rb
index dcf457b9d63..e87707953ae 100644
--- a/lib/gitlab/database/partitioning_migration_helpers/backfill_partitioned_table.rb
+++ b/lib/gitlab/database/partitioning_migration_helpers/backfill_partitioned_table.rb
@@ -21,7 +21,7 @@ module Gitlab
return
end
- bulk_copy = BulkCopy.new(source_table, partitioned_table, source_column, connection: connection)
+ bulk_copy = Gitlab::Database::PartitioningMigrationHelpers::BulkCopy.new(source_table, partitioned_table, source_column, connection: connection)
parent_batch_relation = relation_scoped_to_range(source_table, source_column, start_id, stop_id)
parent_batch_relation.each_batch(of: SUB_BATCH_SIZE) do |sub_batch|
@@ -56,41 +56,6 @@ module Gitlab
def mark_jobs_as_succeeded(*arguments)
BackgroundMigrationJob.mark_all_as_succeeded(self.class.name, arguments)
end
-
- # Helper class to copy data between two tables via upserts
- class BulkCopy
- DELIMITER = ', '
-
- attr_reader :source_table, :destination_table, :source_column, :connection
-
- def initialize(source_table, destination_table, source_column, connection:)
- @source_table = source_table
- @destination_table = destination_table
- @source_column = source_column
- @connection = connection
- end
-
- def copy_between(start_id, stop_id)
- connection.execute(<<~SQL)
- INSERT INTO #{destination_table} (#{column_listing})
- SELECT #{column_listing}
- FROM #{source_table}
- WHERE #{source_column} BETWEEN #{start_id} AND #{stop_id}
- FOR UPDATE
- ON CONFLICT (#{conflict_targets}) DO NOTHING
- SQL
- end
-
- private
-
- def column_listing
- @column_listing ||= connection.columns(source_table).map(&:name).join(DELIMITER)
- end
-
- def conflict_targets
- connection.primary_key(destination_table).join(DELIMITER)
- end
- end
end
end
end
diff --git a/lib/gitlab/database/partitioning_migration_helpers/bulk_copy.rb b/lib/gitlab/database/partitioning_migration_helpers/bulk_copy.rb
new file mode 100644
index 00000000000..b8f5a2e3ad4
--- /dev/null
+++ b/lib/gitlab/database/partitioning_migration_helpers/bulk_copy.rb
@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module PartitioningMigrationHelpers
+ # Helper class to copy data between two tables via upserts
+ class BulkCopy
+ DELIMITER = ', '
+
+ attr_reader :source_table, :destination_table, :source_column, :connection
+
+ def initialize(source_table, destination_table, source_column, connection:)
+ @source_table = source_table
+ @destination_table = destination_table
+ @source_column = source_column
+ @connection = connection
+ end
+
+ def copy_between(start_id, stop_id)
+ connection.execute(<<~SQL)
+ INSERT INTO #{destination_table} (#{column_listing})
+ SELECT #{column_listing}
+ FROM #{source_table}
+ WHERE #{source_column} BETWEEN #{start_id} AND #{stop_id}
+ FOR UPDATE
+ ON CONFLICT (#{conflict_targets}) DO NOTHING
+ SQL
+ end
+
+ private
+
+ def column_listing
+ @column_listing ||= connection.columns(source_table).map(&:name).join(DELIMITER)
+ end
+
+ def conflict_targets
+ connection.primary_keys(destination_table).join(DELIMITER)
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers.rb
index 8849191f356..7d9c12d776e 100644
--- a/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers.rb
+++ b/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers.rb
@@ -32,46 +32,75 @@ module Gitlab
# column - The name of the column to create the foreign key on.
# on_delete - The action to perform when associated data is removed,
# defaults to "CASCADE".
+ # on_update - The action to perform when associated data is updated,
+ # no default value is set.
# name - The name of the foreign key.
+ # validate - Flag that controls whether the new foreign key will be
+ # validated after creation and if it will be added on the parent table.
+ # If the flag is not set, the constraint will only be enforced for new data
+ # in the existing partitions. The helper will need to be called again
+ # with the flag set to `true` to add the foreign key on the parent table
+ # after validating it on all partitions.
+ # `validate: false` should be paired with `prepare_partitioned_async_foreign_key_validation`
+ # reverse_lock_order - Flag that controls whether we should attempt to acquire locks in the reverse
+ # order of the ALTER TABLE. This can be useful in situations where the foreign
+ # key creation could deadlock with another process.
#
- def add_concurrent_partitioned_foreign_key(source, target, column:, on_delete: :cascade, name: nil)
+ def add_concurrent_partitioned_foreign_key(source, target, column:, **options)
assert_not_in_transaction_block(scope: ERROR_SCOPE)
- partition_options = {
- column: column,
- on_delete: on_delete,
+ options.reverse_merge!({
+ target_column: :id,
+ on_delete: :cascade,
+ on_update: nil,
+ name: nil,
+ validate: true,
+ reverse_lock_order: false,
+ column: column
+ })
- # We'll use the same FK name for all partitions and match it to
- # the name used for the partitioned table to follow the convention
- # used by PostgreSQL when adding FKs to new partitions
- name: name.presence || concurrent_partitioned_foreign_key_name(source, column),
+ # We'll use the same FK name for all partitions and match it to
+ # the name used for the partitioned table to follow the convention
+ # used by PostgreSQL when adding FKs to new partitions
+ options[:name] ||= concurrent_partitioned_foreign_key_name(source, column)
+ check_options = options.slice(:column, :on_delete, :on_update, :name)
+ check_options[:primary_key] = options[:target_column]
- # Force the FK validation to true for partitions (and the partitioned table)
- validate: true
- }
-
- if foreign_key_exists?(source, target, **partition_options)
+ if foreign_key_exists?(source, target, **check_options)
warning_message = "Foreign key not created because it exists already " \
"(this may be due to an aborted migration or similar): " \
- "source: #{source}, target: #{target}, column: #{partition_options[:column]}, "\
- "name: #{partition_options[:name]}, on_delete: #{partition_options[:on_delete]}"
+ "source: #{source}, target: #{target}, column: #{options[:column]}, "\
+ "name: #{options[:name]}, on_delete: #{options[:on_delete]}, "\
+ "on_update: #{options[:on_update]}"
Gitlab::AppLogger.warn warning_message
return
end
- partitioned_table = find_partitioned_table(source)
-
- partitioned_table.postgres_partitions.order(:name).each do |partition|
- add_concurrent_foreign_key(partition.identifier, target, **partition_options)
+ Gitlab::Database::PostgresPartitionedTable.each_partition(source) do |partition|
+ add_concurrent_foreign_key(partition.identifier, target, **options)
end
- with_lock_retries do
- add_foreign_key(source, target, **partition_options)
+ # If we are to add the FK on the parent table now, it will trigger
+ # the validation on all partitions. The helper must be called one
+ # more time with `validate: true` after the FK is valid on all partitions.
+ return unless options[:validate]
+
+ options[:allow_partitioned] = true
+ add_concurrent_foreign_key(source, target, **options)
+ end
+
+ def validate_partitioned_foreign_key(source, column, name: nil)
+ assert_not_in_transaction_block(scope: ERROR_SCOPE)
+
+ Gitlab::Database::PostgresPartitionedTable.each_partition(source) do |partition|
+ validate_foreign_key(partition.identifier, column, name: name)
end
end
+ private
+
# Returns the name for a concurrent partitioned foreign key.
#
# Similar to concurrent_foreign_key_name (Gitlab::Database::MigrationHelpers)
diff --git a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb
index f9790bf53b9..61e95dbe1a4 100644
--- a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb
+++ b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb
@@ -6,19 +6,16 @@ module Gitlab
module TableManagementHelpers
include ::Gitlab::Database::SchemaHelpers
include ::Gitlab::Database::MigrationHelpers
+ include ::Gitlab::Database::MigrationHelpers::LooseForeignKeyHelpers
ALLOWED_TABLES = %w[audit_events web_hook_logs].freeze
ERROR_SCOPE = 'table partitioning'
MIGRATION_CLASS_NAME = "::#{module_parent_name}::BackfillPartitionedTable"
+ MIGRATION = "BackfillPartitionedTable"
BATCH_INTERVAL = 2.minutes.freeze
BATCH_SIZE = 50_000
-
- JobArguments = Struct.new(:start_id, :stop_id, :source_table_name, :partitioned_table_name, :source_column) do
- def self.from_array(arguments)
- self.new(*arguments)
- end
- end
+ SUB_BATCH_SIZE = 2_500
# Creates a partitioned copy of an existing table, using a RANGE partitioning strategy on a timestamp column.
# One partition is created per month between the given `min_date` and `max_date`. Also installs a trigger on
@@ -107,12 +104,21 @@ module Gitlab
partitioned_table_name = make_partitioned_table_name(table_name)
primary_key = connection.primary_key(table_name)
- enqueue_background_migration(table_name, partitioned_table_name, primary_key)
+
+ queue_batched_background_migration(
+ MIGRATION,
+ table_name,
+ primary_key,
+ partitioned_table_name,
+ batch_size: BATCH_SIZE,
+ sub_batch_size: SUB_BATCH_SIZE,
+ job_interval: BATCH_INTERVAL
+ )
end
# Cleanup a previously enqueued background migration to copy data into a partitioned table. This will not
# prevent the enqueued jobs from executing, but instead cleans up information in the database used to track the
- # state of the background migration. It should be safe to also remove the partitioned table even if the
+ # state of the batched background migration. It should be safe to also remove the partitioned table even if the
# background jobs are still in-progress, as the absence of the table will cause them to safely exit.
#
# Example:
@@ -122,7 +128,10 @@ module Gitlab
def cleanup_partitioning_data_migration(table_name)
assert_table_is_allowed(table_name)
- cleanup_migration_jobs(table_name)
+ partitioned_table_name = make_partitioned_table_name(table_name)
+ primary_key = connection.primary_key(table_name)
+
+ delete_batched_background_migration(MIGRATION, table_name, primary_key, [partitioned_table_name])
end
def create_hash_partitions(table_name, number_of_partitions)
@@ -142,14 +151,11 @@ module Gitlab
end
end
- # Executes cleanup tasks from a previous BackgroundMigration to backfill a partitioned table by finishing
- # pending jobs and performing a final data synchronization.
- # This performs two steps:
- # 1. Wait to finish any pending BackgroundMigration jobs that have not succeeded
- # 2. Inline copy any missed rows from the original table to the partitioned table
+ # Executes jobs from previous BatchedBackgroundMigration to backfill the partitioned table by finishing
+ # pending jobs.
#
# **NOTE** Migrations using this method cannot be scheduled in the same release as the migration that
- # schedules the background migration using the `enqueue_background_migration` helper, or else the
+ # schedules the background migration using the `enqueue_partitioning_data_migration` helper, or else the
# background migration jobs will be force-executed.
#
# Example:
@@ -157,23 +163,21 @@ module Gitlab
# finalize_backfilling_partitioned_table :audit_events
#
def finalize_backfilling_partitioned_table(table_name)
- Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_dml_mode!
-
assert_table_is_allowed(table_name)
- assert_not_in_transaction_block(scope: ERROR_SCOPE)
partitioned_table_name = make_partitioned_table_name(table_name)
+
unless table_exists?(partitioned_table_name)
raise "could not find partitioned table for #{table_name}, " \
"this could indicate the previous partitioning migration has been rolled back."
end
- Gitlab::BackgroundMigration.steal(MIGRATION_CLASS_NAME) do |background_job|
- JobArguments.from_array(background_job.args.second).source_table_name == table_name.to_s
- end
-
- primary_key = connection.primary_key(table_name)
- copy_missed_records(table_name, partitioned_table_name, primary_key)
+ ensure_batched_background_migration_is_finished(
+ job_class_name: MIGRATION,
+ table_name: table_name,
+ column_name: connection.primary_key(table_name),
+ job_arguments: [partitioned_table_name]
+ )
Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.with_suppressed do
disable_statement_timeout do
@@ -251,22 +255,22 @@ module Gitlab
create_sync_trigger(source_table_name, trigger_name, function_name)
end
- def prepare_constraint_for_list_partitioning(table_name:, partitioning_column:, parent_table_name:, initial_partitioning_value:)
+ def prepare_constraint_for_list_partitioning(table_name:, partitioning_column:, parent_table_name:, initial_partitioning_value:, async: false)
validate_not_in_transaction!(:prepare_constraint_for_list_partitioning)
- Gitlab::Database::Partitioning::ConvertTableToFirstListPartition
+ Gitlab::Database::Partitioning::List::ConvertTable
.new(migration_context: self,
table_name: table_name,
parent_table_name: parent_table_name,
partitioning_column: partitioning_column,
zero_partition_value: initial_partitioning_value
- ).prepare_for_partitioning
+ ).prepare_for_partitioning(async: async)
end
def revert_preparing_constraint_for_list_partitioning(table_name:, partitioning_column:, parent_table_name:, initial_partitioning_value:)
validate_not_in_transaction!(:revert_preparing_constraint_for_list_partitioning)
- Gitlab::Database::Partitioning::ConvertTableToFirstListPartition
+ Gitlab::Database::Partitioning::List::ConvertTable
.new(migration_context: self,
table_name: table_name,
parent_table_name: parent_table_name,
@@ -278,7 +282,7 @@ module Gitlab
def convert_table_to_first_list_partition(table_name:, partitioning_column:, parent_table_name:, initial_partitioning_value:, lock_tables: [])
validate_not_in_transaction!(:convert_table_to_first_list_partition)
- Gitlab::Database::Partitioning::ConvertTableToFirstListPartition
+ Gitlab::Database::Partitioning::List::ConvertTable
.new(migration_context: self,
table_name: table_name,
parent_table_name: parent_table_name,
@@ -291,7 +295,7 @@ module Gitlab
def revert_converting_table_to_first_list_partition(table_name:, partitioning_column:, parent_table_name:, initial_partitioning_value:)
validate_not_in_transaction!(:revert_converting_table_to_first_list_partition)
- Gitlab::Database::Partitioning::ConvertTableToFirstListPartition
+ Gitlab::Database::Partitioning::List::ConvertTable
.new(migration_context: self,
table_name: table_name,
parent_table_name: parent_table_name,
@@ -444,45 +448,6 @@ module Gitlab
create_trigger(table_name, trigger_name, function_name, fires: 'AFTER INSERT OR UPDATE OR DELETE')
end
- def enqueue_background_migration(source_table_name, partitioned_table_name, source_column)
- source_model = define_batchable_model(source_table_name)
-
- queue_background_migration_jobs_by_range_at_intervals(
- source_model,
- MIGRATION_CLASS_NAME,
- BATCH_INTERVAL,
- batch_size: BATCH_SIZE,
- other_job_arguments: [source_table_name.to_s, partitioned_table_name, source_column],
- track_jobs: true)
- end
-
- def cleanup_migration_jobs(table_name)
- ::Gitlab::Database::BackgroundMigrationJob.for_partitioning_migration(MIGRATION_CLASS_NAME, table_name).delete_all
- end
-
- def copy_missed_records(source_table_name, partitioned_table_name, source_column)
- backfill_table = BackfillPartitionedTable.new(connection: connection)
-
- relation = ::Gitlab::Database::BackgroundMigrationJob.pending
- .for_partitioning_migration(MIGRATION_CLASS_NAME, source_table_name)
-
- relation.each_batch do |batch|
- batch.each do |pending_migration_job|
- job_arguments = JobArguments.from_array(pending_migration_job.arguments)
- start_id = job_arguments.start_id
- stop_id = job_arguments.stop_id
-
- say("Backfilling data into partitioned table for ids from #{start_id} to #{stop_id}")
- job_updated_count = backfill_table.perform(start_id, stop_id, source_table_name,
- partitioned_table_name, source_column)
-
- unless job_updated_count > 0
- raise "failed to update tracking record for ids from #{start_id} to #{stop_id}"
- end
- end
- end
- end
-
def replace_table(original_table_name, replacement_table_name, replaced_table_name, primary_key_name)
replace_table = Gitlab::Database::Partitioning::ReplaceTable.new(connection,
original_table_name.to_s, replacement_table_name, replaced_table_name, primary_key_name)
diff --git a/lib/gitlab/database/pg_depend.rb b/lib/gitlab/database/pg_depend.rb
new file mode 100644
index 00000000000..5f3cdaeb979
--- /dev/null
+++ b/lib/gitlab/database/pg_depend.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ class PgDepend < SharedModel
+ self.table_name = 'pg_depend'
+
+ TYPES = {
+ 'VIEW' => %w[v m].freeze
+ }.freeze
+
+ scope :from_pg_extension, ->(type = nil) do
+ joins('INNER JOIN pg_class ON pg_class.oid = pg_depend.objid')
+ .where(pg_class: { relkind: TYPES.fetch(type.to_s) })
+ .where("refclassid = 'pg_extension'::pg_catalog.regclass")
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/postgres_foreign_key.rb b/lib/gitlab/database/postgres_foreign_key.rb
index 04ef574a451..bb3e1d45f15 100644
--- a/lib/gitlab/database/postgres_foreign_key.rb
+++ b/lib/gitlab/database/postgres_foreign_key.rb
@@ -5,6 +5,8 @@ module Gitlab
class PostgresForeignKey < SharedModel
self.primary_key = :oid
+ has_many :child_foreign_keys, class_name: 'Gitlab::Database::PostgresForeignKey', foreign_key: 'parent_oid'
+
# These values come from the possible confdeltype / confupdtype values in pg_constraint
ACTION_TYPES = {
restrict: 'r',
@@ -38,6 +40,14 @@ module Gitlab
scope :by_constrained_table_name, ->(name) { where(constrained_table_name: name) }
+ scope :by_constrained_table_name_or_identifier, ->(name) do
+ if name =~ Database::FULLY_QUALIFIED_IDENTIFIER
+ by_constrained_table_identifier(name)
+ else
+ by_constrained_table_name(name)
+ end
+ end
+
scope :not_inherited, -> { where(is_inherited: false) }
scope :by_name, ->(name) { where(name: name) }
diff --git a/lib/gitlab/database/postgres_partition.rb b/lib/gitlab/database/postgres_partition.rb
index 36dc6818157..e63c6fc86ea 100644
--- a/lib/gitlab/database/postgres_partition.rb
+++ b/lib/gitlab/database/postgres_partition.rb
@@ -7,6 +7,8 @@ module Gitlab
belongs_to :postgres_partitioned_table, foreign_key: 'parent_identifier', primary_key: 'identifier'
+ # identifier includes the partition schema.
+ # For example 'gitlab_partitions_static.events_03', or 'gitlab_partitions_dynamic.logs_03'
scope :for_identifier, ->(identifier) do
unless identifier =~ Gitlab::Database::FULLY_QUALIFIED_IDENTIFIER
raise ArgumentError, "Partition name is not fully qualified with a schema: #{identifier}"
@@ -19,8 +21,12 @@ module Gitlab
for_identifier(identifier).first!
end
- scope :for_parent_table, ->(name) do
- where("parent_identifier = concat(current_schema(), '.', ?)", name).order(:name)
+ scope :for_parent_table, ->(parent_table) do
+ if parent_table =~ Database::FULLY_QUALIFIED_IDENTIFIER
+ where(parent_identifier: parent_table).order(:name)
+ else
+ where("parent_identifier = concat(current_schema(), '.', ?)", parent_table).order(:name)
+ end
end
def self.partition_exists?(table_name)
diff --git a/lib/gitlab/database/query_analyzers/gitlab_schemas_metrics.rb b/lib/gitlab/database/query_analyzers/gitlab_schemas_metrics.rb
index b4b9161f0c2..133933b6ccd 100644
--- a/lib/gitlab/database/query_analyzers/gitlab_schemas_metrics.rb
+++ b/lib/gitlab/database/query_analyzers/gitlab_schemas_metrics.rb
@@ -21,7 +21,7 @@ module Gitlab
db_config_name = ::Gitlab::Database.db_config_name(parsed.connection)
return unless db_config_name
- gitlab_schemas = ::Gitlab::Database::GitlabSchema.table_schemas(parsed.pg.tables)
+ gitlab_schemas = ::Gitlab::Database::GitlabSchema.table_schemas!(parsed.pg.tables)
return if gitlab_schemas.empty?
# to reduce amount of labels sort schemas used
diff --git a/lib/gitlab/database/query_analyzers/gitlab_schemas_validate_connection.rb b/lib/gitlab/database/query_analyzers/gitlab_schemas_validate_connection.rb
index c966ae0e105..976f8c3e7e4 100644
--- a/lib/gitlab/database/query_analyzers/gitlab_schemas_validate_connection.rb
+++ b/lib/gitlab/database/query_analyzers/gitlab_schemas_validate_connection.rb
@@ -15,7 +15,7 @@ module Gitlab
def analyze(parsed)
tables = parsed.pg.select_tables + parsed.pg.dml_tables
- table_schemas = ::Gitlab::Database::GitlabSchema.table_schemas(tables)
+ table_schemas = ::Gitlab::Database::GitlabSchema.table_schemas!(tables)
return if table_schemas.empty?
allowed_schemas = ::Gitlab::Database.gitlab_schemas_for_connection(parsed.connection)
diff --git a/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb b/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb
index 713e1f772e3..d15a0eaa44c 100644
--- a/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb
+++ b/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb
@@ -22,12 +22,27 @@ module Gitlab
self.with_suppressed(false, &blk)
end
+ # This method will temporary ignore the given tables in a current transaction
+ # This is meant to disable `PreventCrossDB` check for some well known failures
+ def self.temporary_ignore_tables_in_transaction(tables, url:, &blk)
+ return yield unless context&.dig(:ignored_tables)
+
+ begin
+ prev_ignored_tables = context[:ignored_tables]
+ context[:ignored_tables] = prev_ignored_tables + tables
+ yield
+ ensure
+ context[:ignored_tables] = prev_ignored_tables
+ end
+ end
+
def self.begin!
super
context.merge!({
transaction_depth_by_db: Hash.new { |h, k| h[k] = 0 },
- modified_tables_by_db: Hash.new { |h, k| h[k] = Set.new }
+ modified_tables_by_db: Hash.new { |h, k| h[k] = Set.new },
+ ignored_tables: []
})
end
@@ -57,7 +72,7 @@ module Gitlab
if context[:transaction_depth_by_db][database] == 0
context[:modified_tables_by_db][database].clear
- # Attempt to troubleshoot https://gitlab.com/gitlab-org/gitlab/-/issues/351531
+ # Attempt to troubleshoot https://gitlab.com/gitlab-org/gitlab/-/issues/351531
::CrossDatabaseModification::TransactionStackTrackRecord.log_gitlab_transactions_stack(action: :end_of_transaction)
elsif context[:transaction_depth_by_db][database] < 0
context[:transaction_depth_by_db][database] = 0
@@ -79,6 +94,9 @@ module Gitlab
# https://gitlab.com/gitlab-org/gitlab/-/issues/343394
tables -= %w[plans gitlab_subscriptions]
+ # Ignore some tables
+ tables -= context[:ignored_tables].to_a
+
return if tables.empty?
# All migrations will write to schema_migrations in the same transaction.
@@ -88,7 +106,7 @@ module Gitlab
context[:modified_tables_by_db][database].merge(tables)
all_tables = context[:modified_tables_by_db].values.flat_map(&:to_a)
- schemas = ::Gitlab::Database::GitlabSchema.table_schemas(all_tables)
+ schemas = ::Gitlab::Database::GitlabSchema.table_schemas!(all_tables)
schemas += ApplicationRecord.gitlab_transactions_stack
@@ -97,10 +115,6 @@ module Gitlab
"a transaction modifying the '#{all_tables.to_a.join(", ")}' tables. " \
"Please refer to https://docs.gitlab.com/ee/development/database/multiple_databases.html#removing-cross-database-transactions for details on how to resolve this exception."
- if schemas.any? { |s| s.to_s.start_with?("undefined") }
- message += " The gitlab_schema was undefined for one or more of the tables in this transaction. Any new tables must be added to lib/gitlab/database/gitlab_schemas.yml ."
- end
-
raise CrossDatabaseModificationAcrossUnsupportedTablesError, message
end
rescue CrossDatabaseModificationAcrossUnsupportedTablesError => e
diff --git a/lib/gitlab/database/query_analyzers/restrict_allowed_schemas.rb b/lib/gitlab/database/query_analyzers/restrict_allowed_schemas.rb
index 4ae3622479f..f3e0fc26946 100644
--- a/lib/gitlab/database/query_analyzers/restrict_allowed_schemas.rb
+++ b/lib/gitlab/database/query_analyzers/restrict_allowed_schemas.rb
@@ -100,7 +100,7 @@ module Gitlab
end
def dml_schemas(tables)
- extra_schemas = ::Gitlab::Database::GitlabSchema.table_schemas(tables)
+ extra_schemas = ::Gitlab::Database::GitlabSchema.table_schemas!(tables)
SCHEMA_MAPPING.each do |schema, mapped_schema|
next unless extra_schemas.delete?(schema)
diff --git a/lib/gitlab/database/reindexing.rb b/lib/gitlab/database/reindexing.rb
index 78de7161a0f..739e573b6c4 100644
--- a/lib/gitlab/database/reindexing.rb
+++ b/lib/gitlab/database/reindexing.rb
@@ -28,7 +28,7 @@ module Gitlab
# Hack: Before we do actual reindexing work, create async indexes
Gitlab::Database::AsyncIndexes.create_pending_indexes! if Feature.enabled?(:database_async_index_creation, type: :ops)
Gitlab::Database::AsyncIndexes.drop_pending_indexes!
- Gitlab::Database::AsyncForeignKeys.validate_pending_entries! if Feature.enabled?(:database_async_foreign_key_validation, type: :ops)
+ Gitlab::Database::AsyncConstraints.validate_pending_entries! if Feature.enabled?(:database_async_foreign_key_validation, type: :ops)
automatic_reindexing
end
diff --git a/lib/gitlab/database/schema_helpers.rb b/lib/gitlab/database/schema_helpers.rb
index d81ff4ff1ae..3ae696a71d8 100644
--- a/lib/gitlab/database/schema_helpers.rb
+++ b/lib/gitlab/database/schema_helpers.rb
@@ -31,8 +31,8 @@ module Gitlab
end
def trigger_exists?(table_name, name)
- connection.select_value(<<~SQL)
- SELECT 1
+ result = connection.select_value(<<~SQL.squish)
+ SELECT true
FROM pg_catalog.pg_trigger trgr
INNER JOIN pg_catalog.pg_class rel
ON trgr.tgrelid = rel.oid
@@ -42,6 +42,8 @@ module Gitlab
AND rel.relname = #{connection.quote(table_name)}
AND trgr.tgname = #{connection.quote(name)}
SQL
+
+ !!result
end
def drop_function(name, if_exists: true)
diff --git a/lib/gitlab/database/schema_validation/adapters/column_database_adapter.rb b/lib/gitlab/database/schema_validation/adapters/column_database_adapter.rb
new file mode 100644
index 00000000000..32d638380ea
--- /dev/null
+++ b/lib/gitlab/database/schema_validation/adapters/column_database_adapter.rb
@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module SchemaValidation
+ module Adapters
+ class ColumnDatabaseAdapter
+ def initialize(query_result)
+ @query_result = query_result
+ end
+
+ def name
+ @name ||= query_result['column_name']
+ end
+
+ def table_name
+ query_result['table_name']
+ end
+
+ def data_type
+ query_result['data_type']
+ end
+
+ def default
+ return unless query_result['column_default']
+
+ return if name == 'id' || query_result['column_default'].include?('nextval')
+
+ "DEFAULT #{query_result['column_default']}"
+ end
+
+ def nullable
+ 'NOT NULL' if query_result['not_null']
+ end
+
+ def partition_key?
+ query_result['partition_key']
+ end
+
+ private
+
+ attr_reader :query_result
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/schema_validation/adapters/column_structure_sql_adapter.rb b/lib/gitlab/database/schema_validation/adapters/column_structure_sql_adapter.rb
new file mode 100644
index 00000000000..420195d89dd
--- /dev/null
+++ b/lib/gitlab/database/schema_validation/adapters/column_structure_sql_adapter.rb
@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module SchemaValidation
+ module Adapters
+ UndefinedPGType = Class.new(StandardError)
+
+ class ColumnStructureSqlAdapter
+ NOT_NULL_CONSTR = :CONSTR_NOTNULL
+ DEFAULT_CONSTR = :CONSTR_DEFAULT
+
+ MAPPINGS = {
+ 't' => 'true',
+ 'f' => 'false'
+ }.freeze
+
+ attr_reader :table_name
+
+ def initialize(table_name, pg_query_stmt, partitioning_stmt)
+ @table_name = table_name
+ @pg_query_stmt = pg_query_stmt
+ @partitioning_stmt = partitioning_stmt
+ end
+
+ def name
+ @name ||= pg_query_stmt.colname
+ end
+
+ def data_type
+ type(pg_query_stmt.type_name)
+ end
+
+ def default
+ return if name == 'id'
+
+ value = parse_node(constraints.find { |node| node.constraint.contype == DEFAULT_CONSTR })
+
+ return unless value
+
+ "DEFAULT #{value}"
+ end
+
+ def nullable
+ 'NOT NULL' if constraints.any? { |node| node.constraint.contype == NOT_NULL_CONSTR }
+ end
+
+ def partition_key?
+ partition_keys.include?(name)
+ end
+
+ private
+
+ attr_reader :pg_query_stmt, :partitioning_stmt
+
+ def constraints
+ @constraints ||= pg_query_stmt.constraints
+ end
+
+ # Returns the node type
+ #
+ # pg_type:: type alias, used internally by postgres, +int4+, +int8+, +bool+, +varchar+
+ # type:: type name, like +integer+, +bigint+, +boolean+, +character varying+.
+ # array_ext:: adds the +[]+ extension for array types.
+ # precision_ext:: adds the precision, if have any, like +(255)+, +(6)+.
+ #
+ # @info +timestamp+ and +timestamptz+ have a particular case when precision is defined.
+ # In this case, the order of the statement needs to be re-arranged from
+ # timestamp without time zone(6) to timestamp(6) without a time zone.
+ def type(node)
+ pg_type = parse_node(node.names.last)
+ type = PgTypes::TYPES.fetch(pg_type).dup
+ array_ext = '[]' if node.array_bounds.any?
+ precision_ext = "(#{node.typmods.map { |typmod| parse_node(typmod) }.join(',')})" if node.typmods.any?
+
+ if %w[timestamp timestamptz].include?(pg_type)
+ type.gsub!('timestamp', ['timestamp', precision_ext].compact.join(''))
+ precision_ext = nil
+ end
+
+ [type, precision_ext, array_ext].compact.join('')
+ rescue KeyError => exception
+ raise UndefinedPGType, exception.message
+ end
+
+ # Parses PGQuery nodes recursively
+ #
+ # :constraint:: nodes that groups column default info
+ # :partition_elem:: node that store partition key info
+ # :func_cal:: nodes that stores functions, like +now()+
+ # :a_const:: nodes that stores constant values, like +t+, +f+, +0.0.0.0+, +255+, +1.0+
+ # :type_cast:: nodes that stores casting values, like +'name'::text+, +'0.0.0.0'::inet+
+ # else:: extract node values in the last iteration of the recursion, like +int4+, +1.0+, +now+, +255+
+ #
+ # @note boolean types types are mapped from +t+, +f+ to +true+, +false+
+ def parse_node(node)
+ return unless node
+
+ case node.node
+ when :constraint
+ parse_node(node.constraint.raw_expr)
+ when :partition_elem
+ node.partition_elem.name
+ when :func_call
+ "#{parse_node(node.func_call.funcname.first)}()"
+ when :a_const
+ parse_node(node.a_const.val)
+ when :type_cast
+ value = parse_node(node.type_cast.arg)
+ type = type(node.type_cast.type_name)
+ separator = MAPPINGS.key?(value) ? '' : "::#{type}"
+
+ [MAPPINGS.fetch(value, "'#{value}'"), separator].compact.join('')
+ else
+ node.to_h[node.node].values.last
+ end
+ end
+
+ def partition_keys
+ return [] unless partitioning_stmt
+
+ @partition_keys ||= partitioning_stmt.part_params.map { |key_stmt| parse_node(key_stmt) }
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/schema_validation/database.rb b/lib/gitlab/database/schema_validation/database.rb
index dfc845f0b44..7a0e348a27b 100644
--- a/lib/gitlab/database/schema_validation/database.rb
+++ b/lib/gitlab/database/schema_validation/database.rb
@@ -4,6 +4,8 @@ module Gitlab
module Database
module SchemaValidation
class Database
+ STATIC_PARTITIONS_SCHEMA = 'gitlab_partitions_static'
+
def initialize(connection)
@connection = connection
end
@@ -12,29 +14,113 @@ module Gitlab
index_map[index_name]
end
+ def fetch_trigger_by_name(trigger_name)
+ trigger_map[trigger_name]
+ end
+
+ def fetch_table_by_name(table_name)
+ table_map[table_name]
+ end
+
+ def index_exists?(index_name)
+ index_map[index_name].present?
+ end
+
+ def trigger_exists?(trigger_name)
+ trigger_map[trigger_name].present?
+ end
+
+ def table_exists?(table_name)
+ fetch_table_by_name(table_name).present?
+ end
+
def indexes
index_map.values
end
+ def triggers
+ trigger_map.values
+ end
+
+ def tables
+ table_map.values
+ end
+
private
+ attr_reader :connection
+
+ def schemas
+ @schemas ||= [STATIC_PARTITIONS_SCHEMA, connection.current_schema]
+ end
+
def index_map
@index_map ||=
fetch_indexes.transform_values! do |index_stmt|
- Index.new(PgQuery.parse(index_stmt).tree.stmts.first.stmt.index_stmt)
+ SchemaObjects::Index.new(PgQuery.parse(index_stmt).tree.stmts.first.stmt.index_stmt)
end
end
- attr_reader :connection
+ def trigger_map
+ @trigger_map ||=
+ fetch_triggers.transform_values! do |trigger_stmt|
+ SchemaObjects::Trigger.new(PgQuery.parse(trigger_stmt).tree.stmts.first.stmt.create_trig_stmt)
+ end
+ end
+
+ def table_map
+ @table_map ||= fetch_tables.transform_values! do |stmt|
+ columns = stmt.map { |column| SchemaObjects::Column.new(Adapters::ColumnDatabaseAdapter.new(column)) }
+
+ SchemaObjects::Table.new(stmt.first['table_name'], columns)
+ end
+ end
def fetch_indexes
sql = <<~SQL
SELECT indexname, indexdef
FROM pg_indexes
- WHERE indexname NOT LIKE '%_pkey' AND schemaname IN ('public', 'gitlab_partitions_static');
+ WHERE indexname NOT LIKE '%_pkey' AND schemaname IN ($1, $2);
+ SQL
+
+ connection.select_rows(sql, nil, schemas).to_h
+ end
+
+ def fetch_triggers
+ sql = <<~SQL
+ SELECT triggers.tgname, pg_get_triggerdef(triggers.oid)
+ FROM pg_catalog.pg_trigger triggers
+ INNER JOIN pg_catalog.pg_class rel ON triggers.tgrelid = rel.oid
+ INNER JOIN pg_catalog.pg_namespace nsp ON nsp.oid = rel.relnamespace
+ WHERE triggers.tgisinternal IS FALSE
+ AND nsp.nspname IN ($1, $2)
+ SQL
+
+ connection.select_rows(sql, nil, schemas).to_h
+ end
+
+ def fetch_tables
+ sql = <<~SQL
+ SELECT
+ table_information.relname AS table_name,
+ col_information.attname AS column_name,
+ col_information.attnotnull AS not_null,
+ col_information.attnum = ANY(pg_partitioned_table.partattrs) as partition_key,
+ format_type(col_information.atttypid, col_information.atttypmod) AS data_type,
+ pg_get_expr(col_default_information.adbin, col_default_information.adrelid) AS column_default
+ FROM pg_attribute AS col_information
+ JOIN pg_class AS table_information ON col_information.attrelid = table_information.oid
+ JOIN pg_namespace AS schema_information ON table_information.relnamespace = schema_information.oid
+ LEFT JOIN pg_partitioned_table ON pg_partitioned_table.partrelid = table_information.oid
+ LEFT JOIN pg_attrdef AS col_default_information ON col_information.attrelid = col_default_information.adrelid
+ AND col_information.attnum = col_default_information.adnum
+ WHERE NOT col_information.attisdropped
+ AND col_information.attnum > 0
+ AND table_information.relkind IN ('r', 'p')
+ AND schema_information.nspname IN ($1, $2)
SQL
- @fetch_indexes ||= connection.exec_query(sql).rows.to_h
+ connection.exec_query(sql, nil, schemas).group_by { |row| row['table_name'] }
end
end
end
diff --git a/lib/gitlab/database/schema_validation/inconsistency.rb b/lib/gitlab/database/schema_validation/inconsistency.rb
new file mode 100644
index 00000000000..766f48ef339
--- /dev/null
+++ b/lib/gitlab/database/schema_validation/inconsistency.rb
@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module SchemaValidation
+ class Inconsistency
+ def initialize(validator_class, structure_sql_object, database_object)
+ @validator_class = validator_class
+ @structure_sql_object = structure_sql_object
+ @database_object = database_object
+ end
+
+ def error_message
+ format(validator_class::ERROR_MESSAGE, object_name)
+ end
+
+ def type
+ validator_class.name.demodulize.underscore
+ end
+
+ def object_type
+ structure_sql_object&.class&.name&.demodulize || database_object&.class&.name&.demodulize
+ end
+
+ def table_name
+ structure_sql_object&.table_name || database_object&.table_name
+ end
+
+ def object_name
+ structure_sql_object&.name || database_object&.name
+ end
+
+ def diff
+ Diffy::Diff.new(structure_sql_statement, database_statement)
+ end
+
+ def inspect
+ <<~MSG
+ #{'-' * 54}
+ #{error_message}
+ Diff:
+ #{diff.to_s(:color)}
+ #{'-' * 54}
+ MSG
+ end
+
+ def structure_sql_statement
+ return unless structure_sql_object
+
+ "#{structure_sql_object.statement}\n"
+ end
+
+ def database_statement
+ return unless database_object
+
+ "#{database_object.statement}\n"
+ end
+
+ private
+
+ attr_reader :validator_class, :structure_sql_object, :database_object
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/schema_validation/inconsistency_filter.rb b/lib/gitlab/database/schema_validation/inconsistency_filter.rb
new file mode 100644
index 00000000000..aa3a71c0edb
--- /dev/null
+++ b/lib/gitlab/database/schema_validation/inconsistency_filter.rb
@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module SchemaValidation
+ class InconsistencyFilter
+ def initialize(tables, triggers)
+ @tables = tables
+ @triggers = triggers
+ end
+
+ def to_proc
+ proc do |inconsistency|
+ inconsistency unless ignored?(inconsistency)
+ end
+ end
+
+ private
+
+ attr_reader :tables, :triggers
+
+ def ignored?(inconsistency)
+ case inconsistency.type
+ in 'extra_tables' | 'missing_tables'
+ ignored_table?(inconsistency.table_name)
+ in 'extra_triggers' | 'missing_triggers'
+ ignored_trigger?(inconsistency.object_name)
+ else
+ false
+ end
+ end
+
+ def ignored_table?(name)
+ tables.include?(name)
+ end
+
+ def ignored_trigger?(name)
+ triggers.any? { |ignored_object| name.to_s.include?(ignored_object) }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/schema_validation/index.rb b/lib/gitlab/database/schema_validation/index.rb
deleted file mode 100644
index af0d5f31f4e..00000000000
--- a/lib/gitlab/database/schema_validation/index.rb
+++ /dev/null
@@ -1,25 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Database
- module SchemaValidation
- class Index
- def initialize(parsed_stmt)
- @parsed_stmt = parsed_stmt
- end
-
- def name
- parsed_stmt.idxname
- end
-
- def statement
- @statement ||= PgQuery.deparse_stmt(parsed_stmt)
- end
-
- private
-
- attr_reader :parsed_stmt
- end
- end
- end
-end
diff --git a/lib/gitlab/database/schema_validation/indexes.rb b/lib/gitlab/database/schema_validation/indexes.rb
deleted file mode 100644
index b7c3705bde9..00000000000
--- a/lib/gitlab/database/schema_validation/indexes.rb
+++ /dev/null
@@ -1,37 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Database
- module SchemaValidation
- class Indexes
- def initialize(structure_sql, database)
- @structure_sql = structure_sql
- @database = database
- end
-
- def missing_indexes
- structure_sql.indexes.map(&:name) - database.indexes.map(&:name)
- end
-
- def extra_indexes
- database.indexes.map(&:name) - structure_sql.indexes.map(&:name)
- end
-
- def wrong_indexes
- structure_sql.indexes.filter_map do |structure_sql_index|
- database_index = database.fetch_index_by_name(structure_sql_index.name)
-
- next if database_index.nil?
- next if database_index.statement == structure_sql_index.statement
-
- structure_sql_index.name
- end
- end
-
- private
-
- attr_reader :structure_sql, :database
- end
- end
- end
-end
diff --git a/lib/gitlab/database/schema_validation/pg_types.rb b/lib/gitlab/database/schema_validation/pg_types.rb
new file mode 100644
index 00000000000..0a1999d056e
--- /dev/null
+++ b/lib/gitlab/database/schema_validation/pg_types.rb
@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module SchemaValidation
+ class PgTypes
+ TYPES = {
+ 'bool' => 'boolean',
+ 'bytea' => 'bytea',
+ 'char' => '"char"',
+ 'int8' => 'bigint',
+ 'int2' => 'smallint',
+ 'int4' => 'integer',
+ 'regproc' => 'regproc',
+ 'text' => 'text',
+ 'oid' => 'oid',
+ 'tid' => 'tid',
+ 'xid' => 'xid',
+ 'cid' => 'cid',
+ 'json' => 'json',
+ 'xml' => 'xml',
+ 'pg_node_tree' => 'pg_node_tree',
+ 'pg_ndistinct' => 'pg_ndistinct',
+ 'pg_dependencies' => 'pg_dependencies',
+ 'pg_mcv_list' => 'pg_mcv_list',
+ 'xid8' => 'xid8',
+ 'path' => 'path',
+ 'polygon' => 'polygon',
+ 'float4' => 'real',
+ 'float8' => 'double precision',
+ 'circle' => 'circle',
+ 'money' => 'money',
+ 'macaddr' => 'macaddr',
+ 'inet' => 'inet',
+ 'cidr' => 'cidr',
+ 'macaddr8' => 'macaddr8',
+ 'aclitem' => 'aclitem',
+ 'bpchar' => 'character',
+ 'varchar' => 'character varying',
+ 'date' => 'date',
+ 'time' => 'time without time zone',
+ 'timestamp' => 'timestamp without time zone',
+ 'timestamptz' => 'timestamp with time zone',
+ 'interval' => 'interval',
+ 'timetz' => 'time with time zone',
+ 'bit' => 'bit',
+ 'varbit' => 'bit varying',
+ 'numeric' => 'numeric',
+ 'refcursor' => 'refcursor',
+ 'regprocedure' => 'regprocedure',
+ 'regoper' => 'regoper',
+ 'regoperator' => 'regoperator',
+ 'regclass' => 'regclass',
+ 'regcollation' => 'regcollation',
+ 'regtype' => 'regtype',
+ 'regrole' => 'regrole',
+ 'regnamespace' => 'regnamespace',
+ 'uuid' => 'uuid',
+ 'pg_lsn' => 'pg_lsn',
+ 'tsvector' => 'tsvector',
+ 'gtsvector' => 'gtsvector',
+ 'tsquery' => 'tsquery',
+ 'regconfig' => 'regconfig',
+ 'regdictionary' => 'regdictionary',
+ 'jsonb' => 'jsonb',
+ 'jsonpath' => 'jsonpath',
+ 'txid_snapshot' => 'txid_snapshot',
+ 'pg_snapshot' => 'pg_snapshot'
+ }.freeze
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/schema_validation/runner.rb b/lib/gitlab/database/schema_validation/runner.rb
new file mode 100644
index 00000000000..7a02c8a16d6
--- /dev/null
+++ b/lib/gitlab/database/schema_validation/runner.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module SchemaValidation
+ class Runner
+ def initialize(structure_sql, database, validators: Validators::BaseValidator.all_validators)
+ @structure_sql = structure_sql
+ @database = database
+ @validators = validators
+ end
+
+ def execute
+ validators.flat_map { |c| c.new(structure_sql, database).execute }
+ end
+
+ private
+
+ attr_reader :structure_sql, :database, :validators
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/schema_validation/schema_inconsistency.rb b/lib/gitlab/database/schema_validation/schema_inconsistency.rb
new file mode 100644
index 00000000000..6f50603e784
--- /dev/null
+++ b/lib/gitlab/database/schema_validation/schema_inconsistency.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module SchemaValidation
+ class SchemaInconsistency < ApplicationRecord
+ self.table_name = :schema_inconsistencies
+
+ belongs_to :issue
+
+ validates :object_name, :valitador_name, :table_name, presence: true
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/schema_validation/schema_objects/base.rb b/lib/gitlab/database/schema_validation/schema_objects/base.rb
new file mode 100644
index 00000000000..43d30dc54ae
--- /dev/null
+++ b/lib/gitlab/database/schema_validation/schema_objects/base.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module SchemaValidation
+ module SchemaObjects
+ class Base
+ def initialize(parsed_stmt)
+ @parsed_stmt = parsed_stmt
+ end
+
+ def name
+ raise NoMethodError, "subclasses of #{self.class.name} must implement #{__method__}"
+ end
+
+ def table_name
+ parsed_stmt.relation.relname
+ end
+
+ def statement
+ @statement ||= PgQuery.deparse_stmt(parsed_stmt)
+ end
+
+ private
+
+ attr_reader :parsed_stmt
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/schema_validation/schema_objects/column.rb b/lib/gitlab/database/schema_validation/schema_objects/column.rb
new file mode 100644
index 00000000000..bd219300a13
--- /dev/null
+++ b/lib/gitlab/database/schema_validation/schema_objects/column.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module SchemaValidation
+ module SchemaObjects
+ class Column
+ def initialize(adapter)
+ @adapter = adapter
+ end
+
+ attr_reader :adapter
+
+ delegate :name, :table_name, :partition_key?, to: :adapter
+
+ def statement
+ [name, adapter.data_type, adapter.default, adapter.nullable].compact.join(' ')
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/schema_validation/schema_objects/index.rb b/lib/gitlab/database/schema_validation/schema_objects/index.rb
new file mode 100644
index 00000000000..28d61b18266
--- /dev/null
+++ b/lib/gitlab/database/schema_validation/schema_objects/index.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module SchemaValidation
+ module SchemaObjects
+ class Index < Base
+ def name
+ parsed_stmt.idxname
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/schema_validation/schema_objects/table.rb b/lib/gitlab/database/schema_validation/schema_objects/table.rb
new file mode 100644
index 00000000000..de2e675d20e
--- /dev/null
+++ b/lib/gitlab/database/schema_validation/schema_objects/table.rb
@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module SchemaValidation
+ module SchemaObjects
+ class Table
+ def initialize(name, columns)
+ @name = name
+ @columns = columns
+ end
+
+ attr_reader :name, :columns
+
+ def table_name
+ name
+ end
+
+ def statement
+ format('CREATE TABLE %s (%s)', name, columns_statement)
+ end
+
+ def fetch_column_by_name(column_name)
+ columns.find { |column| column.name == column_name }
+ end
+
+ def column_exists?(column_name)
+ fetch_column_by_name(column_name).present?
+ end
+
+ private
+
+ def columns_statement
+ columns.reject(&:partition_key?).map(&:statement).join(', ')
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/schema_validation/schema_objects/trigger.rb b/lib/gitlab/database/schema_validation/schema_objects/trigger.rb
new file mode 100644
index 00000000000..508e6b27ed3
--- /dev/null
+++ b/lib/gitlab/database/schema_validation/schema_objects/trigger.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module SchemaValidation
+ module SchemaObjects
+ class Trigger < Base
+ def name
+ parsed_stmt.trigname
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/schema_validation/structure_sql.rb b/lib/gitlab/database/schema_validation/structure_sql.rb
index 32c69a0e5e7..299db1c2a7e 100644
--- a/lib/gitlab/database/schema_validation/structure_sql.rb
+++ b/lib/gitlab/database/schema_validation/structure_sql.rb
@@ -4,29 +4,82 @@ module Gitlab
module Database
module SchemaValidation
class StructureSql
- def initialize(structure_file_path)
+ DEFAULT_SCHEMA = 'public'
+
+ def initialize(structure_file_path, schema_name = DEFAULT_SCHEMA)
@structure_file_path = structure_file_path
+ @schema_name = schema_name
+ end
+
+ def index_exists?(index_name)
+ indexes.find { |index| index.name == index_name }.present?
+ end
+
+ def trigger_exists?(trigger_name)
+ triggers.find { |trigger| trigger.name == trigger_name }.present?
+ end
+
+ def fetch_table_by_name(table_name)
+ tables.find { |table| table.name == table_name }
+ end
+
+ def table_exists?(table_name)
+ fetch_table_by_name(table_name).present?
end
def indexes
- @indexes ||= index_statements.map do |index_statement|
- index_statement.relation.schemaname = "public" if index_statement.relation.schemaname == ''
+ @indexes ||= map_with_default_schema(index_statements, SchemaObjects::Index)
+ end
+
+ def triggers
+ @triggers ||= map_with_default_schema(trigger_statements, SchemaObjects::Trigger)
+ end
- Index.new(index_statement)
+ def tables
+ @tables ||= table_statements.map do |stmt|
+ table_name = stmt.relation.relname
+ partition_stmt = stmt.partspec
+
+ columns = stmt.table_elts.select { |n| n.node == :column_def }.map do |column|
+ adapter = Adapters::ColumnStructureSqlAdapter.new(table_name, column.column_def, partition_stmt)
+ SchemaObjects::Column.new(adapter)
+ end
+
+ SchemaObjects::Table.new(table_name, columns)
end
end
private
- attr_reader :structure_file_path
+ attr_reader :structure_file_path, :schema_name
def index_statements
- parsed_structure_file.tree.stmts.filter_map { |s| s.stmt.index_stmt }
+ statements.filter_map { |s| s.stmt.index_stmt }
+ end
+
+ def trigger_statements
+ statements.filter_map { |s| s.stmt.create_trig_stmt }
+ end
+
+ def table_statements
+ statements.filter_map { |s| s.stmt.create_stmt }
+ end
+
+ def statements
+ @statements ||= parsed_structure_file.tree.stmts
end
def parsed_structure_file
PgQuery.parse(File.read(structure_file_path))
end
+
+ def map_with_default_schema(statements, validation_class)
+ statements.map do |statement|
+ statement.relation.schemaname = schema_name if statement.relation.schemaname == ''
+
+ validation_class.new(statement)
+ end
+ end
end
end
end
diff --git a/lib/gitlab/database/schema_validation/track_inconsistency.rb b/lib/gitlab/database/schema_validation/track_inconsistency.rb
new file mode 100644
index 00000000000..47c3492971c
--- /dev/null
+++ b/lib/gitlab/database/schema_validation/track_inconsistency.rb
@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module SchemaValidation
+ class TrackInconsistency
+ def initialize(inconsistency, project, user)
+ @inconsistency = inconsistency
+ @project = project
+ @user = user
+ end
+
+ def execute
+ return unless Gitlab.com?
+ return if inconsistency_record.present?
+
+ result = ::Issues::CreateService.new(container: project, current_user: user, params: params,
+ spam_params: nil).execute
+
+ track_inconsistency(result[:issue]) if result.success?
+ end
+
+ private
+
+ attr_reader :inconsistency, :project, :user
+
+ def track_inconsistency(issue)
+ schema_inconsistency_model.create(
+ issue: issue,
+ object_name: inconsistency.object_name,
+ table_name: inconsistency.table_name,
+ valitador_name: inconsistency.type
+ )
+ end
+
+ def params
+ {
+ title: issue_title,
+ description: issue_description,
+ issue_type: 'issue',
+ labels: %w[database database-inconsistency-report]
+ }
+ end
+
+ def issue_title
+ "New schema inconsistency: #{inconsistency.object_name}"
+ end
+
+ def issue_description
+ <<~MSG
+ We have detected a new schema inconsistency.
+
+ **Table name:** #{inconsistency.table_name}\
+ **Object name:** #{inconsistency.object_name}\
+ **Validator name:** #{inconsistency.type}\
+ **Object type:** #{inconsistency.object_type}\
+ **Error message:** #{inconsistency.error_message}
+
+
+ **Structure.sql statement:**
+
+ ```sql
+ #{inconsistency.structure_sql_statement}
+ ```
+
+ **Database statement:**
+
+ ```sql
+ #{inconsistency.database_statement}
+ ```
+
+ **Diff:**
+
+ ```diff
+ #{inconsistency.diff}
+
+ ```
+
+
+ For more information, please contact the database team.
+ MSG
+ end
+
+ def schema_inconsistency_model
+ Gitlab::Database::SchemaValidation::SchemaInconsistency
+ end
+
+ def inconsistency_record
+ schema_inconsistency_model.find_by(
+ object_name: inconsistency.object_name,
+ table_name: inconsistency.table_name,
+ valitador_name: inconsistency.type
+ )
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/schema_validation/validators/base_validator.rb b/lib/gitlab/database/schema_validation/validators/base_validator.rb
new file mode 100644
index 00000000000..58e0bf5292b
--- /dev/null
+++ b/lib/gitlab/database/schema_validation/validators/base_validator.rb
@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module SchemaValidation
+ module Validators
+ class BaseValidator
+ ERROR_MESSAGE = 'A schema inconsistency has been found'
+
+ def initialize(structure_sql, database)
+ @structure_sql = structure_sql
+ @database = database
+ end
+
+ def self.all_validators
+ [
+ ExtraTables,
+ ExtraTableColumns,
+ ExtraIndexes,
+ ExtraTriggers,
+ MissingTables,
+ MissingTableColumns,
+ MissingIndexes,
+ MissingTriggers,
+ DifferentDefinitionTables,
+ DifferentDefinitionIndexes,
+ DifferentDefinitionTriggers
+ ]
+ end
+
+ def execute
+ raise NoMethodError, "subclasses of #{self.class.name} must implement #{__method__}"
+ end
+
+ private
+
+ attr_reader :structure_sql, :database
+
+ def build_inconsistency(validator_class, structure_sql_object, database_object)
+ Inconsistency.new(validator_class, structure_sql_object, database_object)
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/schema_validation/validators/different_definition_indexes.rb b/lib/gitlab/database/schema_validation/validators/different_definition_indexes.rb
new file mode 100644
index 00000000000..ba12b3cdc61
--- /dev/null
+++ b/lib/gitlab/database/schema_validation/validators/different_definition_indexes.rb
@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module SchemaValidation
+ module Validators
+ class DifferentDefinitionIndexes < BaseValidator
+ ERROR_MESSAGE = "The %s index has a different statement between structure.sql and database"
+
+ def execute
+ structure_sql.indexes.filter_map do |structure_sql_index|
+ database_index = database.fetch_index_by_name(structure_sql_index.name)
+
+ next if database_index.nil?
+ next if database_index.statement == structure_sql_index.statement
+
+ build_inconsistency(self.class, structure_sql_index, database_index)
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/schema_validation/validators/different_definition_tables.rb b/lib/gitlab/database/schema_validation/validators/different_definition_tables.rb
new file mode 100644
index 00000000000..9fbddbd3fcd
--- /dev/null
+++ b/lib/gitlab/database/schema_validation/validators/different_definition_tables.rb
@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module SchemaValidation
+ module Validators
+ class DifferentDefinitionTables < BaseValidator
+ ERROR_MESSAGE = "The table %s has a different column statement between structure.sql and database"
+
+ def execute
+ structure_sql.tables.filter_map do |structure_sql_table|
+ table_name = structure_sql_table.name
+ database_table = database.fetch_table_by_name(table_name)
+
+ next unless database_table
+
+ db_diffs, structure_diffs = column_diffs(database_table, structure_sql_table.columns)
+
+ if db_diffs.any?
+ build_inconsistency(self.class,
+ SchemaObjects::Table.new(table_name, db_diffs),
+ SchemaObjects::Table.new(table_name, structure_diffs))
+ end
+ end
+ end
+
+ private
+
+ def column_diffs(db_table, columns)
+ db_diffs = []
+ structure_diffs = []
+
+ columns.each do |column|
+ db_column = db_table.fetch_column_by_name(column.name)
+
+ next unless db_column
+
+ next if db_column.statement == column.statement
+
+ db_diffs << db_column
+ structure_diffs << column
+ end
+
+ [db_diffs, structure_diffs]
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/schema_validation/validators/different_definition_triggers.rb b/lib/gitlab/database/schema_validation/validators/different_definition_triggers.rb
new file mode 100644
index 00000000000..79ffe9a6a98
--- /dev/null
+++ b/lib/gitlab/database/schema_validation/validators/different_definition_triggers.rb
@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module SchemaValidation
+ module Validators
+ class DifferentDefinitionTriggers < BaseValidator
+ ERROR_MESSAGE = "The %s trigger has a different statement between structure.sql and database"
+
+ def execute
+ structure_sql.triggers.filter_map do |structure_sql_trigger|
+ database_trigger = database.fetch_trigger_by_name(structure_sql_trigger.name)
+
+ next if database_trigger.nil?
+ next if database_trigger.statement == structure_sql_trigger.statement
+
+ build_inconsistency(self.class, structure_sql_trigger, nil)
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/schema_validation/validators/extra_indexes.rb b/lib/gitlab/database/schema_validation/validators/extra_indexes.rb
new file mode 100644
index 00000000000..c8d3749894b
--- /dev/null
+++ b/lib/gitlab/database/schema_validation/validators/extra_indexes.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module SchemaValidation
+ module Validators
+ class ExtraIndexes < BaseValidator
+ ERROR_MESSAGE = "The index %s is present in the database, but not in the structure.sql file"
+
+ def execute
+ database.indexes.filter_map do |database_index|
+ next if structure_sql.index_exists?(database_index.name)
+
+ build_inconsistency(self.class, nil, database_index)
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/schema_validation/validators/extra_table_columns.rb b/lib/gitlab/database/schema_validation/validators/extra_table_columns.rb
new file mode 100644
index 00000000000..823b01cf808
--- /dev/null
+++ b/lib/gitlab/database/schema_validation/validators/extra_table_columns.rb
@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module SchemaValidation
+ module Validators
+ class ExtraTableColumns < BaseValidator
+ ERROR_MESSAGE = "The table %s has columns present in the database, but not in the structure.sql file"
+
+ def execute
+ database.tables.filter_map do |database_table|
+ table_name = database_table.name
+ structure_sql_table = structure_sql.fetch_table_by_name(table_name)
+
+ next unless structure_sql_table
+
+ inconsistencies = database_table.columns.filter_map do |database_table_column|
+ next if structure_sql_table.column_exists?(database_table_column.name)
+
+ database_table_column
+ end
+
+ if inconsistencies.any?
+ build_inconsistency(self.class, nil, SchemaObjects::Table.new(table_name, inconsistencies))
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/schema_validation/validators/extra_tables.rb b/lib/gitlab/database/schema_validation/validators/extra_tables.rb
new file mode 100644
index 00000000000..99e98eb8f67
--- /dev/null
+++ b/lib/gitlab/database/schema_validation/validators/extra_tables.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module SchemaValidation
+ module Validators
+ class ExtraTables < BaseValidator
+ ERROR_MESSAGE = "The table %s is present in the database, but not in the structure.sql file"
+
+ def execute
+ database.tables.filter_map do |database_table|
+ next if structure_sql.table_exists?(database_table.name)
+
+ build_inconsistency(self.class, nil, database_table)
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/schema_validation/validators/extra_triggers.rb b/lib/gitlab/database/schema_validation/validators/extra_triggers.rb
new file mode 100644
index 00000000000..37dcbc53e2e
--- /dev/null
+++ b/lib/gitlab/database/schema_validation/validators/extra_triggers.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module SchemaValidation
+ module Validators
+ class ExtraTriggers < BaseValidator
+ ERROR_MESSAGE = "The trigger %s is present in the database, but not in the structure.sql file"
+
+ def execute
+ database.triggers.filter_map do |database_trigger|
+ next if structure_sql.trigger_exists?(database_trigger.name)
+
+ build_inconsistency(self.class, nil, database_trigger)
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/schema_validation/validators/missing_indexes.rb b/lib/gitlab/database/schema_validation/validators/missing_indexes.rb
new file mode 100644
index 00000000000..7f81aaccf0f
--- /dev/null
+++ b/lib/gitlab/database/schema_validation/validators/missing_indexes.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module SchemaValidation
+ module Validators
+ class MissingIndexes < BaseValidator
+ ERROR_MESSAGE = "The index %s is missing from the database"
+
+ def execute
+ structure_sql.indexes.filter_map do |structure_sql_index|
+ next if database.index_exists?(structure_sql_index.name)
+
+ build_inconsistency(self.class, structure_sql_index, nil)
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/schema_validation/validators/missing_table_columns.rb b/lib/gitlab/database/schema_validation/validators/missing_table_columns.rb
new file mode 100644
index 00000000000..b49d53823ee
--- /dev/null
+++ b/lib/gitlab/database/schema_validation/validators/missing_table_columns.rb
@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module SchemaValidation
+ module Validators
+ class MissingTableColumns < BaseValidator
+ ERROR_MESSAGE = "The table %s has columns missing from the database"
+
+ def execute
+ structure_sql.tables.filter_map do |structure_sql_table|
+ table_name = structure_sql_table.name
+ database_table = database.fetch_table_by_name(table_name)
+
+ next unless database_table
+
+ inconsistencies = structure_sql_table.columns.filter_map do |structure_table_column|
+ next if database_table.column_exists?(structure_table_column.name)
+
+ structure_table_column
+ end
+
+ if inconsistencies.any?
+ build_inconsistency(self.class, nil, SchemaObjects::Table.new(table_name, inconsistencies))
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/schema_validation/validators/missing_tables.rb b/lib/gitlab/database/schema_validation/validators/missing_tables.rb
new file mode 100644
index 00000000000..f1c9383487d
--- /dev/null
+++ b/lib/gitlab/database/schema_validation/validators/missing_tables.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module SchemaValidation
+ module Validators
+ class MissingTables < BaseValidator
+ ERROR_MESSAGE = "The table %s is missing from the database"
+
+ def execute
+ structure_sql.tables.filter_map do |structure_sql_table|
+ next if database.table_exists?(structure_sql_table.name)
+
+ build_inconsistency(self.class, structure_sql_table, nil)
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/schema_validation/validators/missing_triggers.rb b/lib/gitlab/database/schema_validation/validators/missing_triggers.rb
new file mode 100644
index 00000000000..36236463bbf
--- /dev/null
+++ b/lib/gitlab/database/schema_validation/validators/missing_triggers.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module SchemaValidation
+ module Validators
+ class MissingTriggers < BaseValidator
+ ERROR_MESSAGE = "The trigger %s is missing from the database"
+
+ def execute
+ structure_sql.triggers.filter_map do |structure_sql_trigger|
+ next if database.trigger_exists?(structure_sql_trigger.name)
+
+ build_inconsistency(self.class, structure_sql_trigger, nil)
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/tables_locker.rb b/lib/gitlab/database/tables_locker.rb
index c417ce716e8..0b0d46f4b0e 100644
--- a/lib/gitlab/database/tables_locker.rb
+++ b/lib/gitlab/database/tables_locker.rb
@@ -3,11 +3,12 @@
module Gitlab
module Database
class TablesLocker
- GITLAB_SCHEMAS_TO_IGNORE = %i[gitlab_geo].freeze
+ GITLAB_SCHEMAS_TO_IGNORE = %i[gitlab_embedding gitlab_geo].freeze
def initialize(logger: nil, dry_run: false)
@logger = logger
@dry_run = dry_run
+ @result = []
end
def unlock_writes
@@ -16,11 +17,15 @@ module Gitlab
# TODO: https://gitlab.com/gitlab-org/gitlab/-/issues/366834
next if schema_name.in? GITLAB_SCHEMAS_TO_IGNORE
- lock_writes_manager(table_name, connection, database_name).unlock_writes
+ unlock_writes_on_table(table_name, connection, database_name)
end
end
+
+ @result
end
+ # It locks the tables on the database where they don't belong. Also it unlocks the tables
+ # on the database where they belong
def lock_writes
Gitlab::Database::EachDatabase.each_database_connection(include_shared: false) do |connection, database_name|
schemas_for_connection = Gitlab::Database.gitlab_schemas_for_connection(connection)
@@ -30,16 +35,36 @@ module Gitlab
next if schema_name.in? GITLAB_SCHEMAS_TO_IGNORE
if schemas_for_connection.include?(schema_name)
- lock_writes_manager(table_name, connection, database_name).unlock_writes
+ unlock_writes_on_table(table_name, connection, database_name)
else
- lock_writes_manager(table_name, connection, database_name).lock_writes
+ lock_writes_on_table(table_name, connection, database_name)
end
end
end
+
+ @result
end
private
+ # Unlocks the writes on the table and its partitions
+ def unlock_writes_on_table(table_name, connection, database_name)
+ @result << lock_writes_manager(table_name, connection, database_name).unlock_writes
+
+ table_attached_partitions(table_name, connection) do |postgres_partition|
+ @result << lock_writes_manager(postgres_partition.identifier, connection, database_name).unlock_writes
+ end
+ end
+
+ # It locks the writes on the table and its partitions
+ def lock_writes_on_table(table_name, connection, database_name)
+ @result << lock_writes_manager(table_name, connection, database_name).lock_writes
+
+ table_attached_partitions(table_name, connection) do |postgres_partition|
+ @result << lock_writes_manager(postgres_partition.identifier, connection, database_name).lock_writes
+ end
+ end
+
def tables_to_lock(connection, &block)
Gitlab::Database::GitlabSchema.tables_to_schema.each(&block)
@@ -50,6 +75,14 @@ module Gitlab
end
end
+ def table_attached_partitions(table_name, connection, &block)
+ Gitlab::Database::SharedModel.using_connection(connection) do
+ break unless Gitlab::Database::PostgresPartitionedTable.find_by_name_in_current_schema(table_name)
+
+ Gitlab::Database::PostgresPartitionedTable.each_partition(table_name, &block)
+ end
+ end
+
def lock_writes_manager(table_name, connection, database_name)
Gitlab::Database::LockWritesManager.new(
table_name: table_name,
diff --git a/lib/gitlab/database_importers/instance_administrators/create_group.rb b/lib/gitlab/database_importers/instance_administrators/create_group.rb
deleted file mode 100644
index bb489ced3d2..00000000000
--- a/lib/gitlab/database_importers/instance_administrators/create_group.rb
+++ /dev/null
@@ -1,133 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module DatabaseImporters
- module InstanceAdministrators
- class CreateGroup < ::BaseService
- include Stepable
-
- NAME = 'GitLab Instance'
- PATH_PREFIX = 'gitlab-instance'
- VISIBILITY_LEVEL = Gitlab::VisibilityLevel::INTERNAL
-
- steps :validate_application_settings,
- :validate_admins,
- :create_group,
- :save_group_id,
- :add_group_members,
- :track_event
-
- def initialize
- super(nil)
- end
-
- def execute
- execute_steps
- end
-
- private
-
- def validate_application_settings(result)
- return success(result) if application_settings
-
- log_error('No application_settings found')
- error(_('No application_settings found'))
- end
-
- def validate_admins(result)
- unless instance_admins.any?
- log_error('No active admin user found')
- return error(_('No active admin user found'))
- end
-
- success(result)
- end
-
- def create_group(result)
- if group_created?
- log_info(_('Instance administrators group already exists'))
- result[:group] = instance_administrators_group
- return success(result)
- end
-
- result[:group] = ::Groups::CreateService.new(instance_admins.first, create_group_params).execute
-
- if result[:group].persisted?
- success(result)
- else
- log_error("Could not create instance administrators group. Errors: %{errors}" % { errors: result[:group].errors.full_messages })
- error(_('Could not create group'))
- end
- end
-
- def save_group_id(result)
- return success(result) if group_created?
-
- response = application_settings.update(
- instance_administrators_group_id: result[:group].id
- )
-
- if response
- success(result)
- else
- log_error("Could not save instance administrators group ID, errors: %{errors}" % { errors: application_settings.errors.full_messages })
- error(_('Could not save group ID'))
- end
- end
-
- def add_group_members(result)
- group = result[:group]
- members = group.add_members(members_to_add(group), Gitlab::Access::MAINTAINER)
- errors = members.flat_map { |member| member.errors.full_messages }
-
- if errors.any?
- log_error('Could not add admins as members to self-monitoring project. Errors: %{errors}' % { errors: errors })
- error(_('Could not add admins as members'))
- else
- success(result)
- end
- end
-
- def track_event(result)
- ::Gitlab::Tracking.event("instance_administrators_group", "group_created", namespace: result[:group])
-
- success(result)
- end
-
- def group_created?
- instance_administrators_group.present?
- end
-
- def application_settings
- @application_settings ||= ApplicationSetting.current_without_cache
- end
-
- def instance_administrators_group
- application_settings.instance_administrators_group
- end
-
- def instance_admins
- @instance_admins ||= User.admins.active
- end
-
- def members_to_add(group)
- # Exclude admins who are already members of group because
- # `group.add_members(users)` returns an error if the users parameter contains
- # users who are already members of the group.
- instance_admins - group.members.collect(&:user)
- end
-
- def create_group_params
- {
- name: NAME,
- visibility_level: VISIBILITY_LEVEL,
-
- # The 8 random characters at the end are so that the path does not
- # clash with any existing group that the user might have created.
- path: "#{PATH_PREFIX}-#{SecureRandom.hex(4)}"
- }
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/database_importers/security/training_providers/importer.rb b/lib/gitlab/database_importers/security/training_providers/importer.rb
index aa6a9f29c6d..87bef6400fa 100644
--- a/lib/gitlab/database_importers/security/training_providers/importer.rb
+++ b/lib/gitlab/database_importers/security/training_providers/importer.rb
@@ -20,6 +20,13 @@ module Gitlab
url: "https://integration-api.securecodewarrior.com/api/v1/trial"
}.freeze
+ SECUREFLAG_DATA = {
+ name: 'SecureFlag',
+ description: "Get remediation advice with example code and recommended hands-on labs in a fully
+ interactive virtualised environment.",
+ url: "https://knowledge-base-api.secureflag.com/gitlab"
+ }.freeze
+
module Security
class TrainingProvider < ApplicationRecord
self.table_name = 'security_training_providers'
@@ -31,7 +38,7 @@ module Gitlab
timestamps = { created_at: current_time, updated_at: current_time }
Security::TrainingProvider.upsert_all(
- [KONTRA_DATA.merge(timestamps), SCW_DATA.merge(timestamps)],
+ [KONTRA_DATA.merge(timestamps), SCW_DATA.merge(timestamps), SECUREFLAG_DATA.merge(timestamps)],
unique_by: :index_security_training_providers_on_unique_name
)
end
diff --git a/lib/gitlab/database_importers/self_monitoring/helpers.rb b/lib/gitlab/database_importers/self_monitoring/helpers.rb
deleted file mode 100644
index 6956401e20d..00000000000
--- a/lib/gitlab/database_importers/self_monitoring/helpers.rb
+++ /dev/null
@@ -1,25 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module DatabaseImporters
- module SelfMonitoring
- module Helpers
- def application_settings
- @application_settings ||= ApplicationSetting.current_without_cache
- end
-
- def project_created?
- self_monitoring_project.present?
- end
-
- def self_monitoring_project
- application_settings.self_monitoring_project
- end
-
- def self_monitoring_project_id
- application_settings.self_monitoring_project_id
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/database_importers/self_monitoring/project/create_service.rb b/lib/gitlab/database_importers/self_monitoring/project/create_service.rb
deleted file mode 100644
index be500171bef..00000000000
--- a/lib/gitlab/database_importers/self_monitoring/project/create_service.rb
+++ /dev/null
@@ -1,171 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module DatabaseImporters
- module SelfMonitoring
- module Project
- class CreateService < ::BaseService
- include Stepable
- include SelfMonitoring::Helpers
-
- VISIBILITY_LEVEL = Gitlab::VisibilityLevel::INTERNAL
- PROJECT_NAME = 'Monitoring'
-
- steps :validate_application_settings,
- :create_group,
- :create_project,
- :save_project_id,
- :create_environment,
- :add_prometheus_manual_configuration,
- :track_event
-
- def initialize
- super(nil)
- end
-
- def execute
- execute_steps
- end
-
- private
-
- def validate_application_settings(_result)
- return success if application_settings
-
- log_error('No application_settings found')
- error(_('No application_settings found'))
- end
-
- def create_group(result)
- create_group_response =
- Gitlab::DatabaseImporters::InstanceAdministrators::CreateGroup.new.execute
-
- if create_group_response[:status] == :success
- success(result.merge(create_group_response))
- else
- error(create_group_response[:message])
- end
- end
-
- def create_project(result)
- if project_created?
- log_info('Instance administration project already exists')
- result[:project] = self_monitoring_project
- return success(result)
- end
-
- owner = result[:group].owners.first
-
- result[:project] = ::Projects::CreateService.new(owner, create_project_params(result[:group])).execute
-
- if result[:project].persisted?
- success(result)
- else
- log_error("Could not create instance administration project. Errors: %{errors}" % { errors: result[:project].errors.full_messages })
- error(_('Could not create project'))
- end
- end
-
- def save_project_id(result)
- return success(result) if project_created?
-
- response = application_settings.update(
- self_monitoring_project_id: result[:project].id
- )
-
- if response
- # In the add_prometheus_manual_configuration method, the Prometheus
- # server_address config is saved as an api_url in the Integrations::Prometheus
- # model. There are validates hooks in the Integrations::Prometheus model that
- # check if the project associated with the Integrations::Prometheus is the
- # self_monitoring project. It checks
- # Gitlab::CurrentSettings.self_monitoring_project_id, which is why the
- # Gitlab::CurrentSettings cache needs to be expired here, so that
- # Integrations::Prometheus sees the latest self_monitoring_project_id.
- Gitlab::CurrentSettings.expire_current_application_settings
- success(result)
- else
- log_error("Could not save instance administration project ID, errors: %{errors}" % { errors: application_settings.errors.full_messages })
- error(_('Could not save project ID'))
- end
- end
-
- def create_environment(result)
- return success(result) if result[:project].environments.exists?
-
- environment = ::Environment.new(project_id: result[:project].id, name: 'production')
-
- if environment.save
- success(result)
- else
- log_error("Could not create environment for the Self-monitoring project. Errors: %{errors}" % { errors: environment.errors.full_messages })
- error(_('Could not create environment'))
- end
- end
-
- def add_prometheus_manual_configuration(result)
- return success(result) unless prometheus_enabled?
- return success(result) unless prometheus_server_address.present?
-
- prometheus = result[:project].find_or_initialize_integration('prometheus')
-
- unless prometheus.update(prometheus_integration_attributes)
- log_error('Could not save prometheus manual configuration for self-monitoring project. Errors: %{errors}' % { errors: prometheus.errors.full_messages })
- return error(_('Could not save prometheus manual configuration'))
- end
-
- success(result)
- end
-
- def track_event(result)
- project = result[:project]
- ::Gitlab::Tracking.event("self_monitoring", "project_created", project: project, namespace: project.namespace)
-
- success(result)
- end
-
- def parse_url(uri_string)
- Addressable::URI.parse(uri_string)
- rescue Addressable::URI::InvalidURIError, TypeError
- end
-
- def prometheus_enabled?
- ::Gitlab::Prometheus::Internal.prometheus_enabled?
- end
-
- def prometheus_server_address
- ::Gitlab::Prometheus::Internal.server_address
- end
-
- def docs_path
- Rails.application.routes.url_helpers.help_page_path(
- 'administration/monitoring/gitlab_self_monitoring_project/index'
- )
- end
-
- def create_project_params(group)
- {
- initialize_with_readme: true,
- visibility_level: VISIBILITY_LEVEL,
- name: PROJECT_NAME,
- description: "This project is automatically generated and helps monitor this GitLab instance. [Learn more](#{docs_path}).",
- namespace_id: group.id
- }
- end
-
- def internal_prometheus_server_address_uri
- ::Gitlab::Prometheus::Internal.uri
- end
-
- def prometheus_integration_attributes
- {
- api_url: internal_prometheus_server_address_uri,
- manual_configuration: true,
- active: true
- }
- end
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/database_importers/self_monitoring/project/delete_service.rb b/lib/gitlab/database_importers/self_monitoring/project/delete_service.rb
deleted file mode 100644
index d5bed94d735..00000000000
--- a/lib/gitlab/database_importers/self_monitoring/project/delete_service.rb
+++ /dev/null
@@ -1,46 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module DatabaseImporters
- module SelfMonitoring
- module Project
- class DeleteService < ::BaseService
- include Stepable
- include SelfMonitoring::Helpers
-
- steps :validate_self_monitoring_project_exists,
- :destroy_project
-
- def initialize
- super(nil)
- end
-
- def execute
- execute_steps
- end
-
- private
-
- def validate_self_monitoring_project_exists(result)
- unless project_created? || self_monitoring_project_id.present?
- return error(_('Self-monitoring project does not exist'))
- end
-
- success(result)
- end
-
- def destroy_project(result)
- return success(result) unless project_created?
-
- if self_monitoring_project.destroy
- success(result)
- else
- log_error(self_monitoring_project.errors.full_messages)
- error(_('Error deleting project. Check logs for error details.'))
- end
- end
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/database_importers/work_items/base_type_importer.rb b/lib/gitlab/database_importers/work_items/base_type_importer.rb
index 9796a5905e3..8f8f44e8392 100644
--- a/lib/gitlab/database_importers/work_items/base_type_importer.rb
+++ b/lib/gitlab/database_importers/work_items/base_type_importer.rb
@@ -18,7 +18,10 @@ module Gitlab
progress: 'Progress',
status: 'Status',
requirement_legacy: 'Requirement legacy',
- test_reports: 'Test reports'
+ test_reports: 'Test reports',
+ notifications: 'Notifications',
+ current_user_todos: "Current user todos",
+ award_emoji: 'Award emoji'
}.freeze
WIDGETS_FOR_TYPE = {
@@ -32,23 +35,36 @@ module Gitlab
:notes,
:iteration,
:weight,
- :health_status
+ :health_status,
+ :notifications,
+ :current_user_todos,
+ :award_emoji
],
incident: [
+ :assignees,
:description,
:hierarchy,
- :notes
+ :notes,
+ :notifications,
+ :current_user_todos,
+ :award_emoji
],
test_case: [
:description,
- :notes
+ :notes,
+ :notifications,
+ :current_user_todos,
+ :award_emoji
],
requirement: [
:description,
:notes,
:status,
:requirement_legacy,
- :test_reports
+ :test_reports,
+ :notifications,
+ :current_user_todos,
+ :award_emoji
],
task: [
:assignees,
@@ -59,7 +75,10 @@ module Gitlab
:milestone,
:notes,
:iteration,
- :weight
+ :weight,
+ :notifications,
+ :current_user_todos,
+ :award_emoji
],
objective: [
:assignees,
@@ -69,7 +88,10 @@ module Gitlab
:milestone,
:notes,
:health_status,
- :progress
+ :progress,
+ :notifications,
+ :current_user_todos,
+ :award_emoji
],
key_result: [
:assignees,
@@ -79,7 +101,10 @@ module Gitlab
:start_and_due_date,
:notes,
:health_status,
- :progress
+ :progress,
+ :notifications,
+ :current_user_todos,
+ :award_emoji
]
}.freeze
diff --git a/lib/gitlab/deprecation_json_logger.rb b/lib/gitlab/deprecation_json_logger.rb
index 9796b24868b..5b0900a86dd 100644
--- a/lib/gitlab/deprecation_json_logger.rb
+++ b/lib/gitlab/deprecation_json_logger.rb
@@ -2,6 +2,8 @@
module Gitlab
class DeprecationJsonLogger < Gitlab::JsonLogger
+ exclude_context!
+
def self.file_name_noext
'deprecation_json'
end
diff --git a/lib/gitlab/design_management/copy_design_collection_model_attributes.yml b/lib/gitlab/design_management/copy_design_collection_model_attributes.yml
index 95f15bd6dee..fe1baeb7b67 100644
--- a/lib/gitlab/design_management/copy_design_collection_model_attributes.yml
+++ b/lib/gitlab/design_management/copy_design_collection_model_attributes.yml
@@ -16,6 +16,7 @@
design_attributes:
- filename
- relative_position
+ - description
version_attributes:
- author_id
@@ -30,6 +31,8 @@ ignore_design_attributes:
- issue_id
- project_id
- iid
+ - description_html
+ - cached_markdown_version
ignore_version_attributes:
- id
diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb
index 225b4f7cf86..95ea3fe9f0f 100644
--- a/lib/gitlab/diff/highlight.rb
+++ b/lib/gitlab/diff/highlight.rb
@@ -24,15 +24,15 @@ module Gitlab
end
def highlight
- populate_marker_ranges if Feature.enabled?(:use_marker_ranges, project)
+ populate_marker_ranges
- @diff_lines.map.with_index do |diff_line, index|
+ @diff_lines.map do |diff_line|
diff_line = diff_line.dup
# ignore highlighting for "match" lines
next diff_line if diff_line.meta?
rich_line = apply_syntax_highlight(diff_line)
- rich_line = apply_marker_ranges_highlight(diff_line, rich_line, index)
+ rich_line = apply_marker_ranges_highlight(diff_line, rich_line)
diff_line.rich_text = rich_line
@@ -60,12 +60,8 @@ module Gitlab
highlight_line(diff_line) || ERB::Util.html_escape(diff_line.text)
end
- def apply_marker_ranges_highlight(diff_line, rich_line, index)
- marker_ranges = if Feature.enabled?(:use_marker_ranges, project)
- diff_line.marker_ranges
- else
- inline_diffs[index]
- end
+ def apply_marker_ranges_highlight(diff_line, rich_line)
+ marker_ranges = diff_line.marker_ranges
return rich_line if marker_ranges.blank?
@@ -134,12 +130,6 @@ module Gitlab
end
end
- # Deprecated: https://gitlab.com/gitlab-org/gitlab/-/issues/324638
- # ------------------------------------------------------------------------
- def inline_diffs
- @inline_diffs ||= InlineDiff.for_lines(@raw_lines)
- end
-
def old_lines
@old_lines ||= highlighted_blob_lines(diff_file.old_blob)
end
diff --git a/lib/gitlab/diff/highlight_cache.rb b/lib/gitlab/diff/highlight_cache.rb
index 5128b09aef4..63a437b021d 100644
--- a/lib/gitlab/diff/highlight_cache.rb
+++ b/lib/gitlab/diff/highlight_cache.rb
@@ -71,7 +71,6 @@ module Gitlab
strong_memoize(:redis_key) do
options = [
diff_options,
- Feature.enabled?(:use_marker_ranges, diffable.project),
Feature.enabled?(:diff_line_syntax_highlighting, diffable.project)
]
options_for_key = OpenSSL::Digest::SHA256.hexdigest(options.join)
diff --git a/lib/gitlab/diff/inline_diff.rb b/lib/gitlab/diff/inline_diff.rb
index 802da50cfc6..7f760a23f45 100644
--- a/lib/gitlab/diff/inline_diff.rb
+++ b/lib/gitlab/diff/inline_diff.rb
@@ -17,27 +17,6 @@ module Gitlab
CharDiff.new(old_line, new_line).changed_ranges(offset: offset)
end
-
- # Deprecated: https://gitlab.com/gitlab-org/gitlab/-/issues/324638
- class << self
- def for_lines(lines)
- pair_selector = Gitlab::Diff::PairSelector.new(lines)
-
- inline_diffs = []
-
- pair_selector.each do |old_index, new_index|
- old_line = lines[old_index]
- new_line = lines[new_index]
-
- old_diffs, new_diffs = new(old_line, new_line, offset: 1).inline_diffs
-
- inline_diffs[old_index] = old_diffs
- inline_diffs[new_index] = new_diffs
- end
-
- inline_diffs
- end
- end
end
end
end
diff --git a/lib/gitlab/diff/parser.rb b/lib/gitlab/diff/parser.rb
index b29c75ed467..a506bc3aaa2 100644
--- a/lib/gitlab/diff/parser.rb
+++ b/lib/gitlab/diff/parser.rb
@@ -84,8 +84,6 @@ module Gitlab
"new"
when "-"
"old"
- else
- nil
end
end
end
diff --git a/lib/gitlab/discussions_diff/highlight_cache.rb b/lib/gitlab/discussions_diff/highlight_cache.rb
index 14cb773251b..df93e6e91b4 100644
--- a/lib/gitlab/discussions_diff/highlight_cache.rb
+++ b/lib/gitlab/discussions_diff/highlight_cache.rb
@@ -16,11 +16,11 @@ module Gitlab
def write_multiple(mapping)
with_redis do |redis|
Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do
- redis.multi do |multi|
+ redis.pipelined do |pipelined|
mapping.each do |raw_key, value|
key = cache_key_for(raw_key)
- multi.set(key, gzip_compress(value.to_json), ex: EXPIRATION)
+ pipelined.set(key, gzip_compress(value.to_json), ex: EXPIRATION)
end
end
end
@@ -41,7 +41,13 @@ module Gitlab
content =
with_redis do |redis|
Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do
- redis.mget(keys)
+ if ::Feature.enabled?(:use_pipeline_over_multikey)
+ redis.pipelined do |pipeline|
+ keys.each { |key| pipeline.get(key) }
+ end
+ else
+ redis.mget(keys)
+ end
end
end
@@ -66,7 +72,13 @@ module Gitlab
with_redis do |redis|
Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do
- redis.del(keys)
+ if ::Feature.enabled?(:use_pipeline_over_multikey)
+ redis.pipelined do |pipeline|
+ keys.each { |key| pipeline.del(key) }
+ end.sum
+ else
+ redis.del(keys)
+ end
end
end
end
diff --git a/lib/gitlab/doorkeeper_secret_storing/secret/pbkdf2_sha512.rb b/lib/gitlab/doorkeeper_secret_storing/secret/pbkdf2_sha512.rb
index e0884557496..0624fe934f9 100644
--- a/lib/gitlab/doorkeeper_secret_storing/secret/pbkdf2_sha512.rb
+++ b/lib/gitlab/doorkeeper_secret_storing/secret/pbkdf2_sha512.rb
@@ -10,9 +10,7 @@ module Gitlab
# additional security.
SALT = ''
- def self.transform_secret(plain_secret, stored_as_hash = false)
- return plain_secret if Feature.disabled?(:hash_oauth_secrets) && !stored_as_hash
-
+ def self.transform_secret(plain_secret)
Devise::Pbkdf2Encryptable::Encryptors::Pbkdf2Sha512.digest(plain_secret, STRETCHES, SALT)
end
@@ -28,8 +26,7 @@ module Gitlab
# Securely compare the given +input+ value with a +stored+ value
# processed by +transform_secret+.
def self.secret_matches?(input, stored)
- stored_as_hash = stored.starts_with?('$pbkdf2-')
- transformed_input = transform_secret(input, stored_as_hash)
+ transformed_input = transform_secret(input)
ActiveSupport::SecurityUtils.secure_compare transformed_input, stored
end
end
diff --git a/lib/gitlab/email/handler/create_note_handler.rb b/lib/gitlab/email/handler/create_note_handler.rb
index b168efaac11..e6c64e2b1d6 100644
--- a/lib/gitlab/email/handler/create_note_handler.rb
+++ b/lib/gitlab/email/handler/create_note_handler.rb
@@ -50,7 +50,9 @@ module Gitlab
end
def create_note
- sent_notification.create_reply(note_message)
+ external_author = from_address if author == User.support_bot
+
+ sent_notification.create_reply(note_message, external_author)
end
def note_message
diff --git a/lib/gitlab/email/hook/silent_mode_interceptor.rb b/lib/gitlab/email/hook/silent_mode_interceptor.rb
new file mode 100644
index 00000000000..56f94119472
--- /dev/null
+++ b/lib/gitlab/email/hook/silent_mode_interceptor.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Email
+ module Hook
+ class SilentModeInterceptor
+ def self.delivering_email(message)
+ if Gitlab::CurrentSettings.silent_mode_enabled?
+ message.perform_deliveries = false
+
+ Gitlab::AppJsonLogger.info(
+ message: "SilentModeInterceptor prevented sending mail",
+ mail_subject: message.subject,
+ silent_mode_enabled: true
+ )
+ else
+ Gitlab::AppJsonLogger.debug(
+ message: "SilentModeInterceptor did nothing",
+ mail_subject: message.subject,
+ silent_mode_enabled: false
+ )
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/email/hook/validate_addresses_interceptor.rb b/lib/gitlab/email/hook/validate_addresses_interceptor.rb
deleted file mode 100644
index e63f047e63d..00000000000
--- a/lib/gitlab/email/hook/validate_addresses_interceptor.rb
+++ /dev/null
@@ -1,32 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Email
- module Hook
- # Check for unsafe characters in the envelope-from and -to addresses.
- # These are passed directly as arguments to sendmail and are liable to shell injection attacks:
- # https://github.com/mikel/mail/blob/2.7.1/lib/mail/network/delivery_methods/sendmail.rb#L53-L58
- class ValidateAddressesInterceptor
- UNSAFE_CHARACTERS = /(\\|[^[:print:]])/.freeze
-
- def self.delivering_email(message)
- addresses = Array(message.smtp_envelope_from) + Array(message.smtp_envelope_to)
-
- addresses.each do |address|
- next unless address.match?(UNSAFE_CHARACTERS)
-
- Gitlab::AuthLogger.info(
- message: 'Skipping email with unsafe characters in address',
- address: address,
- subject: message.subject
- )
-
- message.perform_deliveries = false
-
- break
- end
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/email/html_parser.rb b/lib/gitlab/email/html_parser.rb
index 10dbedbb464..693048adabf 100644
--- a/lib/gitlab/email/html_parser.rb
+++ b/lib/gitlab/email/html_parser.rb
@@ -34,11 +34,7 @@ module Gitlab
end
def filtered_text
- @filtered_text ||= if Feature.enabled?(:service_desk_html_to_text_email_handler)
- ::Gitlab::Email::HtmlToMarkdownParser.convert(filtered_html)
- else
- Html2Text.convert(filtered_html)
- end
+ @filtered_text ||= ::Gitlab::Email::HtmlToMarkdownParser.convert(filtered_html)
end
end
end
diff --git a/lib/gitlab/email/html_to_markdown_parser.rb b/lib/gitlab/email/html_to_markdown_parser.rb
index 42dd012308b..5dd3725cc3e 100644
--- a/lib/gitlab/email/html_to_markdown_parser.rb
+++ b/lib/gitlab/email/html_to_markdown_parser.rb
@@ -5,25 +5,46 @@ require 'nokogiri'
module Gitlab
module Email
class HtmlToMarkdownParser < Html2Text
- ADDITIONAL_TAGS = %w[em strong img details].freeze
- IMG_ATTRS = %w[alt src].freeze
+ extend Gitlab::Utils::Override
+ # List of tags to be converted by Markdown.
+ #
+ # All attributes are removed except for the defined ones.
+ #
+ # <tag> => [<attribute to keep>, ...]
+ ALLOWED_TAG_ATTRIBUTES = {
+ 'em' => [],
+ 'strong' => [],
+ 'details' => [],
+ 'img' => %w[alt src]
+ }.freeze
+ private_constant :ALLOWED_TAG_ATTRIBUTES
+
+ # This redefinition can be removed once https://github.com/soundasleep/html2text_ruby/pull/30
+ # is merged and released.
def self.convert(html)
html = fix_newlines(replace_entities(html))
doc = Nokogiri::HTML(html)
- HtmlToMarkdownParser.new(doc).convert
+ new(doc).convert
end
+ private
+
+ override :iterate_over
def iterate_over(node)
- return super unless ADDITIONAL_TAGS.include?(node.name)
+ allowed_attributes = ALLOWED_TAG_ATTRIBUTES[node.name]
+ return super unless allowed_attributes
- if node.name == 'img'
- node.keys.each { |key| node.remove_attribute(key) unless IMG_ATTRS.include?(key) } # rubocop:disable Style/HashEachMethods
- end
+ remove_attributes(node, allowed_attributes)
Kramdown::Document.new(node.to_html, input: 'html').to_commonmark
end
+
+ def remove_attributes(node, allowed_attributes)
+ to_remove = (node.keys - allowed_attributes)
+ to_remove.each { |key| node.remove_attribute(key) }
+ end
end
end
end
diff --git a/lib/gitlab/email/incoming_email.rb b/lib/gitlab/email/incoming_email.rb
new file mode 100644
index 00000000000..a0a01ae0d70
--- /dev/null
+++ b/lib/gitlab/email/incoming_email.rb
@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Email
+ module IncomingEmail
+ class << self
+ include Gitlab::Email::Common
+
+ def config
+ incoming_email_config
+ end
+
+ def key_from_address(address, wildcard_address: nil)
+ wildcard_address ||= config.address
+ regex = address_regex(wildcard_address)
+ return unless regex
+
+ match = address.match(regex)
+ return unless match
+
+ match[1]
+ end
+
+ private
+
+ def address_regex(wildcard_address)
+ return unless wildcard_address
+
+ regex = Regexp.escape(wildcard_address)
+ regex = regex.sub(Regexp.escape(WILDCARD_PLACEHOLDER), '(.+)')
+ Regexp.new(/\A<?#{regex}>?\z/).freeze
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb
index 32794a6c99d..51d250ea98c 100644
--- a/lib/gitlab/email/receiver.rb
+++ b/lib/gitlab/email/receiver.rb
@@ -110,7 +110,7 @@ module Gitlab
when String
# Handle emails from clients which append with commas,
# example clients are Microsoft exchange and iOS app
- Gitlab::IncomingEmail.scan_fallback_references(references)
+ email_class.scan_fallback_references(references)
when nil
[]
end
@@ -177,7 +177,7 @@ module Gitlab
def recipients_from_received_headers
strong_memoize :emails_from_received_headers do
- received.map { |header| header.value[RECEIVED_HEADER_REGEX, 1] }.compact
+ received.filter_map { |header| header.value[RECEIVED_HEADER_REGEX, 1] }
end
end
@@ -203,7 +203,7 @@ module Gitlab
end
def email_class
- Gitlab::IncomingEmail
+ Gitlab::Email::IncomingEmail
end
end
end
diff --git a/lib/gitlab/email/service_desk_email.rb b/lib/gitlab/email/service_desk_email.rb
new file mode 100644
index 00000000000..4ea1c077327
--- /dev/null
+++ b/lib/gitlab/email/service_desk_email.rb
@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Email
+ module ServiceDeskEmail
+ class << self
+ include Gitlab::Email::Common
+
+ def config
+ Gitlab.config.service_desk_email
+ end
+
+ def key_from_address(address)
+ wildcard_address = config&.address
+ return unless wildcard_address
+
+ Gitlab::Email::IncomingEmail.key_from_address(address, wildcard_address: wildcard_address)
+ end
+
+ def address_for_key(key)
+ return if config.address.blank?
+
+ config.address.sub(WILDCARD_PLACEHOLDER, key)
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/email/service_desk_receiver.rb b/lib/gitlab/email/service_desk_receiver.rb
index 6c6eb3b0a65..e286cf1f68c 100644
--- a/lib/gitlab/email/service_desk_receiver.rb
+++ b/lib/gitlab/email/service_desk_receiver.rb
@@ -12,7 +12,7 @@ module Gitlab
end
def email_class
- ::Gitlab::ServiceDeskEmail
+ ::Gitlab::Email::ServiceDeskEmail
end
end
end
diff --git a/lib/gitlab/emoji.rb b/lib/gitlab/emoji.rb
index 2b36b1c99bd..7d47bfe88fe 100644
--- a/lib/gitlab/emoji.rb
+++ b/lib/gitlab/emoji.rb
@@ -15,20 +15,6 @@ module Gitlab
Rails.root.join("public/-/emojis/#{EMOJI_VERSION}")
end
- def emoji_image_tag(name, src)
- image_options = {
- class: 'emoji',
- src: src,
- title: ":#{name}:",
- alt: ":#{name}:",
- height: 20,
- width: 20,
- align: 'absmiddle'
- }
-
- ActionController::Base.helpers.tag(:img, image_options)
- end
-
# CSS sprite fallback takes precedence over image fallback
# @param [TanukiEmoji::Character] emoji
# @param [Hash] options
diff --git a/lib/gitlab/encrypted_incoming_email_command.rb b/lib/gitlab/encrypted_incoming_email_command.rb
index a18382439d6..05fc7cac000 100644
--- a/lib/gitlab/encrypted_incoming_email_command.rb
+++ b/lib/gitlab/encrypted_incoming_email_command.rb
@@ -8,7 +8,7 @@ module Gitlab
class << self
def encrypted_secrets
- Gitlab::IncomingEmail.encrypted_secrets
+ Gitlab::Email::IncomingEmail.encrypted_secrets
end
def encrypted_file_template
diff --git a/lib/gitlab/encrypted_service_desk_email_command.rb b/lib/gitlab/encrypted_service_desk_email_command.rb
index ece6da7c1b3..1a0317e0da9 100644
--- a/lib/gitlab/encrypted_service_desk_email_command.rb
+++ b/lib/gitlab/encrypted_service_desk_email_command.rb
@@ -8,7 +8,7 @@ module Gitlab
class << self
def encrypted_secrets
- Gitlab::ServiceDeskEmail.encrypted_secrets
+ Gitlab::Email::ServiceDeskEmail.encrypted_secrets
end
def encrypted_file_template
diff --git a/lib/gitlab/etag_caching/router/graphql.rb b/lib/gitlab/etag_caching/router/graphql.rb
index 7a0fb2ac269..92b21a0859d 100644
--- a/lib/gitlab/etag_caching/router/graphql.rb
+++ b/lib/gitlab/etag_caching/router/graphql.rb
@@ -16,12 +16,17 @@ module Gitlab
[
%r(\Apipelines/sha/\w{7,40}\z),
'ci_editor',
- 'pipeline_authoring'
+ 'pipeline_composition'
],
[
%r(\Aon_demand_scan/counts/),
'on_demand_scans',
'dynamic_application_security_testing'
+ ],
+ [
+ %r(\A/projects/.+/-/environments.json\z),
+ 'environment_details',
+ 'continuous_delivery'
]
].map { |attrs| build_graphql_route(*attrs) }.freeze
diff --git a/lib/gitlab/event_store.rb b/lib/gitlab/event_store.rb
index 023c8ace4d9..ce71ee594f2 100644
--- a/lib/gitlab/event_store.rb
+++ b/lib/gitlab/event_store.rb
@@ -60,6 +60,10 @@ module Gitlab
store.subscribe ::MergeRequests::CreateApprovalNoteWorker, to: ::MergeRequests::ApprovedEvent
store.subscribe ::MergeRequests::ResolveTodosAfterApprovalWorker, to: ::MergeRequests::ApprovedEvent
store.subscribe ::MergeRequests::ExecuteApprovalHooksWorker, to: ::MergeRequests::ApprovedEvent
+ store.subscribe ::MergeRequests::SetReviewerReviewedWorker, to: ::MergeRequests::ApprovedEvent
+ store.subscribe ::Ml::ExperimentTracking::AssociateMlCandidateToPackageWorker,
+ to: ::Packages::PackageCreatedEvent,
+ if: -> (event) { ::Ml::ExperimentTracking::AssociateMlCandidateToPackageWorker.handles_event?(event) }
end
private_class_method :configure!
end
diff --git a/lib/gitlab/exception_log_formatter.rb b/lib/gitlab/exception_log_formatter.rb
index ce802b562f0..52ad67d6f8b 100644
--- a/lib/gitlab/exception_log_formatter.rb
+++ b/lib/gitlab/exception_log_formatter.rb
@@ -17,6 +17,10 @@ module Gitlab
payload['exception.backtrace'] = Rails.backtrace_cleaner.clean(exception.backtrace)
end
+ if exception.cause
+ payload['exception.cause_class'] = exception.cause.class.name
+ end
+
if sql = find_sql(exception)
payload['exception.sql'] = sql
end
diff --git a/lib/gitlab/favicon.rb b/lib/gitlab/favicon.rb
index 8e48b482462..f4633473a95 100644
--- a/lib/gitlab/favicon.rb
+++ b/lib/gitlab/favicon.rb
@@ -24,7 +24,7 @@ module Gitlab
'favicon-blue.png'
end
- def status_overlay(status_name)
+ def ci_status_overlay(status_name)
path = File.join(
'ci_favicons',
"#{status_name}.png"
@@ -33,6 +33,15 @@ module Gitlab
ActionController::Base.helpers.image_path(path, host: host)
end
+ def mr_status_overlay(status_name)
+ path = File.join(
+ 'mr_favicons',
+ "#{status_name}.png"
+ )
+
+ ActionController::Base.helpers.image_path(path, host: host)
+ end
+
def available_status_names
@available_status_names ||= Dir.glob(Rails.root.join('app', 'assets', 'images', 'ci_favicons', '*.png'))
.map { |file| File.basename(file, '.png') }
diff --git a/lib/gitlab/file_finder.rb b/lib/gitlab/file_finder.rb
index 95f896a74e9..8a894901ca1 100644
--- a/lib/gitlab/file_finder.rb
+++ b/lib/gitlab/file_finder.rb
@@ -44,15 +44,11 @@ module Gitlab
# Overridden in Gitlab::WikiFileFinder
def search_paths(query)
- if Feature.enabled?(:code_basic_search_files_by_regexp, project)
- return [] if query.blank? || ref.blank?
-
- escaped_query = RE2::Regexp.escape(query)
- query_regexp = Gitlab::EncodingHelper.encode_utf8_no_detect("(?i)#{escaped_query}")
- repository.search_files_by_regexp(query_regexp, ref)
- else
- repository.search_files_by_name(query, ref)
- end
+ return [] if query.blank? || ref.blank?
+
+ escaped_query = RE2::Regexp.escape(query)
+ query_regexp = Gitlab::EncodingHelper.encode_utf8_no_detect("(?i)#{escaped_query}")
+ repository.search_files_by_regexp(query_regexp, ref)
end
end
end
diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb
index 8e1b51fcec5..ef5c242e68a 100644
--- a/lib/gitlab/git.rb
+++ b/lib/gitlab/git.rb
@@ -1,6 +1,6 @@
# frozen_string_literal: true
-require_dependency 'gitlab/encoding_helper'
+require_relative 'encoding_helper'
module Gitlab
module Git
@@ -22,6 +22,21 @@ module Gitlab
InvalidRefFormatError = Class.new(BaseError)
ReferencesLockedError = Class.new(BaseError)
+ class ResourceExhaustedError < BaseError
+ def initialize(msg = nil, retry_after = 0)
+ super(msg)
+ @retry_after = retry_after
+ end
+
+ def headers
+ if @retry_after.to_i > 0
+ { "Retry-After" => @retry_after }
+ else
+ {}
+ end
+ end
+ end
+
class << self
include Gitlab::EncodingHelper
diff --git a/lib/gitlab/git/blame_mode.rb b/lib/gitlab/git/blame_mode.rb
new file mode 100644
index 00000000000..15200e3888c
--- /dev/null
+++ b/lib/gitlab/git/blame_mode.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Git
+ class BlameMode
+ def initialize(project, params)
+ @project = project
+ @params = params
+ end
+
+ def streaming?
+ Gitlab::Utils.to_boolean(params[:streaming], default: false)
+ end
+
+ def pagination?
+ return false if streaming?
+ return false if Gitlab::Utils.to_boolean(params[:no_pagination], default: false)
+
+ Feature.enabled?(:blame_page_pagination, project)
+ end
+
+ def full?
+ !streaming? && !pagination?
+ end
+
+ private
+
+ attr_reader :project, :params
+ end
+ end
+end
diff --git a/lib/gitlab/git/blame_pagination.rb b/lib/gitlab/git/blame_pagination.rb
new file mode 100644
index 00000000000..6bf29859b14
--- /dev/null
+++ b/lib/gitlab/git/blame_pagination.rb
@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Git
+ class BlamePagination
+ include Gitlab::Utils::StrongMemoize
+
+ PAGINATION_PER_PAGE = 1000
+ STREAMING_FIRST_PAGE_SIZE = 200
+ STREAMING_PER_PAGE = 2000
+
+ def initialize(blob, blame_mode, params)
+ @blob = blob
+ @blame_mode = blame_mode
+ @params = params
+ end
+
+ def page
+ page = params.fetch(:page, 1).to_i
+
+ return 1 if page < 1
+
+ page
+ end
+ strong_memoize_attr :page
+
+ def per_page
+ blame_mode.streaming? ? STREAMING_PER_PAGE : PAGINATION_PER_PAGE
+ end
+ strong_memoize_attr :per_page
+
+ def total_pages
+ total = (blob_lines_count.to_f / per_page).ceil
+ return total unless blame_mode.streaming?
+
+ ([blob_lines_count - STREAMING_FIRST_PAGE_SIZE, 0].max.to_f / per_page).ceil + 1
+ end
+ strong_memoize_attr :total_pages
+
+ def total_extra_pages
+ [total_pages - 1, 0].max
+ end
+ strong_memoize_attr :total_extra_pages
+
+ def paginator
+ return if blame_mode.streaming? || blame_mode.full?
+
+ Kaminari.paginate_array([], total_count: blob_lines_count, limit: per_page)
+ .tap { |pagination| pagination.max_paginates_per(per_page) }
+ .page(page)
+ end
+
+ def blame_range
+ return if blame_mode.full?
+
+ first_line = ((page - 1) * per_page) + 1
+
+ if blame_mode.streaming?
+ return 1..STREAMING_FIRST_PAGE_SIZE if page == 1
+
+ first_line = STREAMING_FIRST_PAGE_SIZE + ((page - 2) * per_page) + 1
+ end
+
+ last_line = (first_line + per_page).to_i - 1
+
+ first_line..last_line
+ end
+
+ private
+
+ attr_reader :blob, :blame_mode, :params
+
+ def blob_lines_count
+ @blob_lines_count ||= blob.data.lines.count
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb
index 267107e04e6..11eb0a584ab 100644
--- a/lib/gitlab/git/commit.rb
+++ b/lib/gitlab/git/commit.rb
@@ -16,7 +16,7 @@ module Gitlab
SERIALIZE_KEYS = [
:id, :message, :parent_ids,
:authored_date, :author_name, :author_email,
- :committed_date, :committer_name, :committer_email, :trailers
+ :committed_date, :committer_name, :committer_email, :trailers, :referenced_by
].freeze
attr_accessor(*SERIALIZE_KEYS)
@@ -414,6 +414,7 @@ module Gitlab
@committer_email = commit.committer.email.dup
@parent_ids = Array(commit.parent_ids)
@trailers = commit.trailers.to_h { |t| [t.key, t.value] }
+ @referenced_by = Array(commit.referenced_by)
end
# Gitaly provides a UNIX timestamp in author.date.seconds, and a timezone
@@ -463,7 +464,8 @@ module Gitlab
end
def fetch_body_from_gitaly
- self.class.get_message(@repository, id)
+ # #to_s is required to ensure BatchLoader is not returned.
+ self.class.get_message(@repository, id).to_s
end
def self.valid?(commit_id)
diff --git a/lib/gitlab/git/diff_collection.rb b/lib/gitlab/git/diff_collection.rb
index 0ffe8bee953..b4dd880ceb7 100644
--- a/lib/gitlab/git/diff_collection.rb
+++ b/lib/gitlab/git/diff_collection.rb
@@ -24,6 +24,8 @@ module Gitlab
limits[:safe_max_lines] = [limits[:max_lines], defaults[:max_lines]].min
limits[:safe_max_bytes] = limits[:safe_max_files] * 5.kilobytes # Average 5 KB per file
limits[:max_patch_bytes] = Gitlab::Git::Diff.patch_hard_limit_bytes
+ limits[:max_patch_bytes_for_file_extension] = options.fetch(:max_patch_bytes_for_file_extension, {})
+
limits
end
diff --git a/lib/gitlab/git/ref.rb b/lib/gitlab/git/ref.rb
index 1d7966a11ed..4a09f866db4 100644
--- a/lib/gitlab/git/ref.rb
+++ b/lib/gitlab/git/ref.rb
@@ -36,8 +36,6 @@ module Gitlab
target.name
elsif target.is_a? String
target
- else
- nil
end
end
end
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index e1399b6642b..80d0fd17568 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -54,8 +54,6 @@ module Gitlab
# state.
alias_method :object_pool_remote_name, :gl_repository
- # This initializer method is only used on the client side (gitlab-ce).
- # Gitaly-ruby uses a different initializer.
def initialize(storage, relative_path, gl_repository, gl_project_path, container: nil)
@storage = storage
@relative_path = relative_path
@@ -263,8 +261,12 @@ module Gitlab
def archive_metadata(ref, storage_path, project_path, format = "tar.gz", append_sha:, path: nil)
ref ||= root_ref
- commit_id = extract_commit_id_from_ref(ref)
- return {} if commit_id.nil?
+ if Feature.enabled?(:resolve_ambiguous_archives, container)
+ commit_id = extract_commit_id_from_ref(ref)
+ return {} if commit_id.nil?
+ else
+ commit_id = ref
+ end
commit = Gitlab::Git::Commit.find(self, commit_id)
return {} if commit.nil?
@@ -807,27 +809,17 @@ module Gitlab
end
end
- def license(from_gitaly)
+ def license
wrapped_gitaly_errors do
response = gitaly_repository_client.find_license
break nil if response.license_short_name.empty?
- if from_gitaly
- break ::Gitlab::Git::DeclaredLicense.new(key: response.license_short_name,
- name: response.license_name,
- nickname: response.license_nickname.presence,
- url: response.license_url.presence,
- path: response.license_path)
- end
-
- licensee_object = Licensee::License.new(response.license_short_name)
-
- break nil if licensee_object.name.blank?
-
- licensee_object.meta.nickname = "LICENSE" if licensee_object.key == "other"
-
- licensee_object
+ ::Gitlab::Git::DeclaredLicense.new(key: response.license_short_name,
+ name: response.license_name,
+ nickname: response.license_nickname.presence,
+ url: response.license_url.presence,
+ path: response.license_path)
end
rescue Licensee::InvalidLicense => e
Gitlab::ErrorTracking.track_exception(e)
@@ -1152,7 +1144,7 @@ module Gitlab
def checksum
# The exists? RPC is much cheaper, so we perform this request first
- raise NoRepository, "Repository does not exists" unless exists?
+ raise NoRepository, "Repository does not exist" unless exists?
gitaly_repository_client.calculate_checksum
rescue GRPC::NotFound
diff --git a/lib/gitlab/git/rugged_impl/tree.rb b/lib/gitlab/git/rugged_impl/tree.rb
index 66cfc02130b..c7a981c7dd4 100644
--- a/lib/gitlab/git/rugged_impl/tree.rb
+++ b/lib/gitlab/git/rugged_impl/tree.rb
@@ -130,7 +130,6 @@ module Gitlab
new(
id: entry[:oid],
- root_id: root_tree.oid,
name: entry[:name],
type: entry[:type],
mode: entry[:filemode].to_s(8),
diff --git a/lib/gitlab/git/tag.rb b/lib/gitlab/git/tag.rb
index 5ed5158eeea..37977a1dfb6 100644
--- a/lib/gitlab/git/tag.rb
+++ b/lib/gitlab/git/tag.rb
@@ -96,8 +96,6 @@ module Gitlab
nil # not implemented, see https://gitlab.com/gitlab-org/gitlab/issues/19260
when :X509
X509::Tag.new(@repository, self).signature
- else
- nil
end
end
diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb
index f0eef619e13..e437f99dab3 100644
--- a/lib/gitlab/git/tree.rb
+++ b/lib/gitlab/git/tree.rb
@@ -6,7 +6,7 @@ module Gitlab
include Gitlab::EncodingHelper
extend Gitlab::Git::WrapsGitalyErrors
- attr_accessor :id, :root_id, :type, :mode, :commit_id, :submodule_url
+ attr_accessor :id, :type, :mode, :commit_id, :submodule_url
attr_writer :name, :path, :flat_path
class << self
@@ -61,7 +61,7 @@ module Gitlab
end
def initialize(options)
- %w(id root_id name path flat_path type mode commit_id).each do |key|
+ %w(id name path flat_path type mode commit_id).each do |key|
self.send("#{key}=", options[key.to_sym]) # rubocop:disable GitlabSecurity/PublicSend
end
end
diff --git a/lib/gitlab/git/wraps_gitaly_errors.rb b/lib/gitlab/git/wraps_gitaly_errors.rb
index 1d34f3c8eb2..20bcf3585e1 100644
--- a/lib/gitlab/git/wraps_gitaly_errors.rb
+++ b/lib/gitlab/git/wraps_gitaly_errors.rb
@@ -5,14 +5,40 @@ module Gitlab
module WrapsGitalyErrors
def wrapped_gitaly_errors(&block)
yield block
- rescue GRPC::NotFound => e
- raise Gitlab::Git::Repository::NoRepository, e
- rescue GRPC::InvalidArgument => e
- raise ArgumentError, e
- rescue GRPC::DeadlineExceeded => e
- raise Gitlab::Git::CommandTimedOut, e
rescue GRPC::BadStatus => e
- raise Gitlab::Git::CommandError, e
+ # The GRPC::BadStatus is the fundamental error that serves as the basis for all other gRPC error categories,
+ # including GRPC::InvalidArgument. It is essential to note that rescuing the specific exception class does not
+ # account for all possible cases. In this regard, a status exception can be directly generated from
+ # GRPC::BadStatus. Therefore, it is recommended that we capture and rescue the GRPC::BadStatus and assert the
+ # status code to ensure adequate coverage of error cases.
+ case e.code
+ when GRPC::Core::StatusCodes::NOT_FOUND
+ raise Gitlab::Git::Repository::NoRepository, e
+ when GRPC::Core::StatusCodes::INVALID_ARGUMENT
+ raise ArgumentError, e
+ when GRPC::Core::StatusCodes::DEADLINE_EXCEEDED
+ raise Gitlab::Git::CommandTimedOut, e
+ when GRPC::Core::StatusCodes::RESOURCE_EXHAUSTED
+ handle_resource_exhausted(e)
+ else
+ raise Gitlab::Git::CommandError, e
+ end
+ end
+
+ private
+
+ def handle_resource_exhausted(exception)
+ detail = Gitlab::GitalyClient.decode_detailed_error(exception)
+
+ case detail.class.name
+ when Gitaly::LimitError.name
+ retry_after = detail&.retry_after&.seconds
+ raise ResourceExhaustedError.new(
+ "Upstream Gitaly has been exhausted: #{detail.error_message}. Try again later", retry_after
+ )
+ else
+ raise ResourceExhaustedError, _("Upstream Gitaly has been exhausted. Try again later")
+ end
end
end
end
diff --git a/lib/gitlab/git_access_design.rb b/lib/gitlab/git_access_design.rb
index bf89c01305a..fb8df0d217a 100644
--- a/lib/gitlab/git_access_design.rb
+++ b/lib/gitlab/git_access_design.rb
@@ -4,6 +4,23 @@ module Gitlab
class GitAccessDesign < GitAccess
extend ::Gitlab::Utils::Override
+ # TODO Re-factor so that correct container is passed to the constructor
+ # and this method can be removed from here
+ # https://gitlab.com/gitlab-org/gitlab/-/issues/409454
+ def initialize(
+ actor, container, protocol, authentication_abilities:, repository_path: nil, redirected_path: nil,
+ auth_result_type: nil)
+ super(
+ actor,
+ select_container(container),
+ protocol,
+ authentication_abilities: authentication_abilities,
+ repository_path: repository_path,
+ redirected_path: redirected_path,
+ auth_result_type: auth_result_type
+ )
+ end
+
def check(_cmd, _changes)
check_protocol!
check_can_create_design!
@@ -18,6 +35,10 @@ module Gitlab
private
+ def select_container(container)
+ container.is_a?(::DesignManagement::Repository) ? container.project : container
+ end
+
def check_protocol!
if protocol != 'web'
raise ::Gitlab::GitAccess::ForbiddenError, "Designs are only accessible using the web interface"
diff --git a/lib/gitlab/git_ref_validator.rb b/lib/gitlab/git_ref_validator.rb
index f4d4cebc096..7867e1b8c37 100644
--- a/lib/gitlab/git_ref_validator.rb
+++ b/lib/gitlab/git_ref_validator.rb
@@ -12,10 +12,10 @@ module Gitlab
# Validates a given name against the git reference specification
#
# Returns true for a valid reference name, false otherwise
- def validate(ref_name)
+ def validate(ref_name, skip_head_ref_check: false)
return false if ref_name.to_s.empty? # #blank? raises an ArgumentError for invalid encodings
return false if ref_name.start_with?(*(EXPANDED_PREFIXES + DISALLOWED_PREFIXES))
- return false if ref_name == 'HEAD'
+ return false if ref_name == 'HEAD' && !skip_head_ref_check
begin
Rugged::Reference.valid_name?("refs/heads/#{ref_name}")
diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb
index 4df9d800ea6..0c67b9fa078 100644
--- a/lib/gitlab/gitaly_client/commit_service.rb
+++ b/lib/gitlab/gitaly_client/commit_service.rb
@@ -146,7 +146,6 @@ module Gitlab
message.entries.map do |gitaly_tree_entry|
Gitlab::Git::Tree.new(
id: gitaly_tree_entry.oid,
- root_id: gitaly_tree_entry.root_oid,
type: gitaly_tree_entry.type.downcase,
mode: gitaly_tree_entry.mode.to_s(8),
name: File.basename(gitaly_tree_entry.path),
@@ -423,7 +422,8 @@ module Gitlab
first_parent: !!options[:first_parent],
global_options: parse_global_options!(options),
disable_walk: true, # This option is deprecated. The 'walk' implementation is being removed.
- trailers: options[:trailers]
+ trailers: options[:trailers],
+ include_referenced_by: options[:include_referenced_by]
)
request.after = GitalyClient.timestamp(options[:after]) if options[:after]
request.before = GitalyClient.timestamp(options[:before]) if options[:before]
@@ -441,14 +441,14 @@ module Gitlab
# revision exists, or `false` otherwise. This function accepts all revisions as specified by
# gitrevisions(1).
def object_existence_map(revisions, gitaly_repo: @gitaly_repo)
+ return {} unless revisions.present?
+
enum = Enumerator.new do |y|
- # This is a bug in Gitaly: revisions of the initial request are ignored. This will be fixed in v15.0 via
- # https://gitlab.com/gitlab-org/gitaly/-/merge_requests/4510, so we can merge initial request and the initial
- # set of revisions starting with v15.1.
- y.yield Gitaly::CheckObjectsExistRequest.new(repository: gitaly_repo)
+ revisions.each_slice(100).with_index do |revisions_subset, i|
+ params = { revisions: revisions_subset }
+ params[:repository] = gitaly_repo if i == 0
- revisions.each_slice(100) do |revisions_subset|
- y.yield Gitaly::CheckObjectsExistRequest.new(revisions: revisions_subset)
+ y.yield Gitaly::CheckObjectsExistRequest.new(**params)
end
end
diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb
index 313334737c0..1af06cc7490 100644
--- a/lib/gitlab/gitaly_client/operation_service.rb
+++ b/lib/gitlab/gitaly_client/operation_service.rb
@@ -45,7 +45,7 @@ module Gitlab
rescue GRPC::BadStatus => e
detailed_error = GitalyClient.decode_detailed_error(e)
- case detailed_error&.error
+ case detailed_error.try(:error)
when :access_check
access_check_error = detailed_error.access_check
# These messages were returned from internal/allowed API calls
@@ -82,7 +82,7 @@ module Gitlab
rescue GRPC::BadStatus => e
detailed_error = GitalyClient.decode_detailed_error(e)
- case detailed_error&.error
+ case detailed_error.try(:error)
when :custom_hook
raise Gitlab::Git::PreReceiveError.new(custom_hook_error_message(detailed_error.custom_hook),
fallback_message: e.details)
@@ -124,7 +124,7 @@ module Gitlab
rescue GRPC::BadStatus => e
detailed_error = GitalyClient.decode_detailed_error(e)
- case detailed_error&.error
+ case detailed_error.try(:error)
when :custom_hook
raise Gitlab::Git::PreReceiveError.new(custom_hook_error_message(detailed_error.custom_hook),
fallback_message: e.details)
@@ -188,7 +188,7 @@ module Gitlab
rescue GRPC::BadStatus => e
detailed_error = GitalyClient.decode_detailed_error(e)
- case detailed_error&.error
+ case detailed_error.try(:error)
when :access_check
access_check_error = detailed_error.access_check
# These messages were returned from internal/allowed API calls
@@ -247,7 +247,7 @@ module Gitlab
rescue GRPC::BadStatus => e
detailed_error = GitalyClient.decode_detailed_error(e)
- case detailed_error&.error
+ case detailed_error.try(:error)
when :access_check
access_check_error = detailed_error.access_check
# These messages were returned from internal/allowed API calls
@@ -329,7 +329,7 @@ module Gitlab
rescue GRPC::BadStatus => e
detailed_error = GitalyClient.decode_detailed_error(e)
- case detailed_error&.error
+ case detailed_error.try(:error)
when :access_check
access_check_error = detailed_error.access_check
# These messages were returned from internal/allowed API calls
@@ -366,7 +366,7 @@ module Gitlab
rescue GRPC::BadStatus => e
detailed_error = GitalyClient.decode_detailed_error(e)
- case detailed_error&.error
+ case detailed_error.try(:error)
when :resolve_revision, :rebase_conflict
# Theoretically, we could now raise specific errors based on the type
# of the detailed error. Most importantly, we get error details when
@@ -458,7 +458,7 @@ module Gitlab
rescue GRPC::BadStatus => e
detailed_error = GitalyClient.decode_detailed_error(e)
- case detailed_error&.error
+ case detailed_error.try(:error)
when :access_check
access_check_error = detailed_error.access_check
# These messages were returned from internal/allowed API calls
diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb
index ac6491e8770..88c79eb8954 100644
--- a/lib/gitlab/gitaly_client/ref_service.rb
+++ b/lib/gitlab/gitaly_client/ref_service.rb
@@ -113,7 +113,7 @@ module Gitlab
rescue GRPC::BadStatus => e
detailed_error = GitalyClient.decode_detailed_error(e)
- case detailed_error&.error
+ case detailed_error.try(:error)
when :tag_not_found
raise Gitlab::Git::UnknownRef, "tag does not exist: #{tag_name}"
else
@@ -135,7 +135,7 @@ module Gitlab
rescue GRPC::BadStatus => e
detailed_error = GitalyClient.decode_detailed_error(e)
- case detailed_error&.error
+ case detailed_error.try(:error)
when :invalid_format
raise Gitlab::Git::InvalidRefFormatError, "references have an invalid format: #{detailed_error.invalid_format.refs.join(",")}"
when :references_locked
@@ -239,7 +239,7 @@ module Gitlab
sort_by = 'name' if sort_by == 'name_asc'
enum_value = Gitaly::FindLocalBranchesRequest::SortBy.resolve(sort_by.upcase.to_sym)
- raise ArgumentError, "Invalid sort_by key `#{sort_by}`" unless enum_value
+ return Gitaly::FindLocalBranchesRequest::SortBy::NAME unless enum_value
enum_value
end
diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb
index bcc03ca08c9..93d58710b0c 100644
--- a/lib/gitlab/gitaly_client/repository_service.rb
+++ b/lib/gitlab/gitaly_client/repository_service.rb
@@ -109,7 +109,7 @@ module Gitlab
# rubocop: enable Metrics/ParameterLists
def create_repository(default_branch = nil)
- request = Gitaly::CreateRepositoryRequest.new(repository: @gitaly_repo, default_branch: default_branch)
+ request = Gitaly::CreateRepositoryRequest.new(repository: @gitaly_repo, default_branch: encode_binary(default_branch))
gitaly_client_call(@storage, :repository_service, :create_repository, request, timeout: GitalyClient.fast_timeout)
end
@@ -306,18 +306,18 @@ module Gitlab
end
def search_files_by_name(ref, query, limit: 0, offset: 0)
- request = Gitaly::SearchFilesByNameRequest.new(repository: @gitaly_repo, ref: ref, query: query, limit: limit, offset: offset)
+ request = Gitaly::SearchFilesByNameRequest.new(repository: @gitaly_repo, ref: encode_binary(ref), query: query, limit: limit, offset: offset)
gitaly_client_call(@storage, :repository_service, :search_files_by_name, request, timeout: GitalyClient.fast_timeout).flat_map(&:files)
end
def search_files_by_content(ref, query, options = {})
- request = Gitaly::SearchFilesByContentRequest.new(repository: @gitaly_repo, ref: ref, query: query)
+ request = Gitaly::SearchFilesByContentRequest.new(repository: @gitaly_repo, ref: encode_binary(ref), query: query)
response = gitaly_client_call(@storage, :repository_service, :search_files_by_content, request, timeout: GitalyClient.default_timeout)
search_results_from_response(response, options)
end
def search_files_by_regexp(ref, filter, limit: 0, offset: 0)
- request = Gitaly::SearchFilesByNameRequest.new(repository: @gitaly_repo, ref: ref, query: '.', filter: filter, limit: limit, offset: offset)
+ request = Gitaly::SearchFilesByNameRequest.new(repository: @gitaly_repo, ref: encode_binary(ref), query: '.', filter: filter, limit: limit, offset: offset)
gitaly_client_call(@storage, :repository_service, :search_files_by_name, request, timeout: GitalyClient.fast_timeout).flat_map(&:files)
end
diff --git a/lib/gitlab/github_import/bulk_importing.rb b/lib/gitlab/github_import/bulk_importing.rb
index 0c91eff1d10..d16f4d7587b 100644
--- a/lib/gitlab/github_import/bulk_importing.rb
+++ b/lib/gitlab/github_import/bulk_importing.rb
@@ -27,8 +27,13 @@ module Gitlab
build_record = model.new(attrs)
if build_record.invalid?
- log_error(object[:id], build_record.errors.full_messages)
- errors << build_record.errors
+ github_identifiers = github_identifiers(object)
+
+ log_error(github_identifiers, build_record.errors.full_messages)
+ errors << {
+ validation_errors: build_record.errors,
+ github_identifiers: github_identifiers
+ }
next
end
@@ -53,17 +58,18 @@ module Gitlab
raise NotImplementedError
end
- def bulk_insert_failures(validation_errors)
- rows = validation_errors.map do |error|
+ def bulk_insert_failures(errors)
+ rows = errors.map do |error|
correlation_id_value = Labkit::Correlation::CorrelationId.current_or_new_id
{
source: self.class.name,
exception_class: 'ActiveRecord::RecordInvalid',
- exception_message: error.full_messages.first.truncate(255),
+ exception_message: error[:validation_errors].full_messages.first.truncate(255),
correlation_id_value: correlation_id_value,
retry_count: nil,
- created_at: Time.zone.now
+ created_at: Time.zone.now,
+ external_identifiers: error[:github_identifiers]
}
end
@@ -88,15 +94,19 @@ module Gitlab
)
end
- def log_error(object_id, messages)
+ def log_error(github_identifiers, messages)
Gitlab::Import::Logger.error(
import_type: :github,
project_id: project.id,
importer: self.class.name,
message: messages,
- github_identifier: object_id
+ github_identifiers: github_identifiers
)
end
+
+ def github_identifiers(object)
+ raise NotImplementedError
+ end
end
end
end
diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb
index 1c9ca9f43a8..886563a6f69 100644
--- a/lib/gitlab/github_import/client.rb
+++ b/lib/gitlab/github_import/client.rb
@@ -112,6 +112,10 @@ module Gitlab
each_object(:branches, *args)
end
+ def collaborators(*args)
+ each_object(:collaborators, *args)
+ end
+
def branch_protection(repo_name, branch_name)
with_rate_limit { octokit.branch_protection(repo_name, branch_name).to_h }
end
diff --git a/lib/gitlab/github_import/clients/proxy.rb b/lib/gitlab/github_import/clients/proxy.rb
index b12df404640..27030f5382a 100644
--- a/lib/gitlab/github_import/clients/proxy.rb
+++ b/lib/gitlab/github_import/clients/proxy.rb
@@ -6,6 +6,10 @@ module Gitlab
class Proxy
attr_reader :client
+ delegate :each_object, :user, :octokit, to: :client
+
+ REPOS_COUNT_CACHE_KEY = 'github-importer/provider-repo-count/%{type}/%{user_id}'
+
def initialize(access_token, client_options)
@client = pick_client(access_token, client_options)
end
@@ -13,24 +17,26 @@ module Gitlab
def repos(search_text, options)
return { repos: filtered(client.repos, search_text) } if use_legacy?
- if use_graphql?
- fetch_repos_via_graphql(search_text, options)
- else
- fetch_repos_via_rest(search_text, options)
- end
+ fetch_repos_via_graphql(search_text, options)
end
- private
+ def count_repos_by(relation_type, user_id)
+ return if use_legacy?
+
+ key = format(REPOS_COUNT_CACHE_KEY, type: relation_type, user_id: user_id)
- def fetch_repos_via_rest(search_text, options)
- { repos: client.search_repos_by_name(search_text, options)[:items] }
+ ::Gitlab::Cache::Import::Caching.read_integer(key, timeout: 5.minutes) ||
+ fetch_and_cache_repos_count_via_graphql(relation_type, key)
end
+ private
+
def fetch_repos_via_graphql(search_text, options)
response = client.search_repos_by_name_graphql(search_text, options)
{
repos: response.dig(:data, :search, :nodes),
- page_info: response.dig(:data, :search, :pageInfo)
+ page_info: response.dig(:data, :search, :pageInfo),
+ count: response.dig(:data, :search, :repositoryCount)
}
end
@@ -50,8 +56,11 @@ module Gitlab
Feature.disabled?(:remove_legacy_github_client)
end
- def use_graphql?
- Feature.enabled?(:github_client_fetch_repos_via_graphql)
+ def fetch_and_cache_repos_count_via_graphql(relation_type, key)
+ response = client.count_repos_by_relation_type_graphql(relation_type: relation_type)
+ count = response.dig(:data, :search, :repositoryCount)
+
+ ::Gitlab::Cache::Import::Caching.write(key, count, timeout: 5.minutes)
end
end
end
diff --git a/lib/gitlab/github_import/clients/search_repos.rb b/lib/gitlab/github_import/clients/search_repos.rb
index b72e5ac7751..a2ef6ca24eb 100644
--- a/lib/gitlab/github_import/clients/search_repos.rb
+++ b/lib/gitlab/github_import/clients/search_repos.rb
@@ -5,24 +5,24 @@ module Gitlab
module Clients
module SearchRepos
def search_repos_by_name_graphql(name, options = {})
- with_retry do
- octokit.post(
- '/graphql',
- { query: graphql_search_repos_body(name, options) }.to_json
- ).to_h
- end
+ graphql_request(graphql_search_repos_body(name, options))
+ end
+
+ def count_repos_by_relation_type_graphql(options)
+ graphql_request(count_by_relation_type_query(options))
end
- def search_repos_by_name(name, options = {})
- search_query = search_repos_query(name, options)
+ private
+ def graphql_request(query)
with_retry do
- octokit.search_repositories(search_query, options).to_h
+ octokit.post(
+ '/graphql',
+ { query: query }.to_json
+ ).to_h
end
end
- private
-
def graphql_search_repos_body(name, options)
query = search_repos_query(name, options)
query = "query: \"#{query}\""
@@ -45,14 +45,15 @@ module Gitlab
endCursor
hasNextPage
hasPreviousPage
- }
+ },
+ repositoryCount
}
}
TEXT
end
def search_repos_query(string, options = {})
- base = "#{string} in:name is:public,private"
+ base = "#{string} in:name is:public,private fork:true"
case options[:relation_type]
when 'organization' then organization_repos_query(base, options)
@@ -64,7 +65,11 @@ module Gitlab
end
def organization_repos_query(search_string, options)
- "#{search_string} org:#{options[:organization_login]}"
+ if options[:organization_login].present?
+ "#{search_string} org:#{options[:organization_login]}"
+ else
+ organizations_subquery
+ end
end
def collaborated_repos_query(search_string)
@@ -95,6 +100,18 @@ module Gitlab
.map { |org| "org:#{org[:login]}" }
.join(' ')
end
+
+ def count_by_relation_type_query(options)
+ query = search_repos_query(nil, options)
+ query = "query: \"#{query}\""
+ <<-TEXT
+ {
+ search(type: REPOSITORY, #{query}) {
+ repositoryCount
+ }
+ }
+ TEXT
+ end
end
end
end
diff --git a/lib/gitlab/github_import/importer/attachments/issues_importer.rb b/lib/gitlab/github_import/importer/attachments/issues_importer.rb
index 090bfb4a098..c8f0b59fd18 100644
--- a/lib/gitlab/github_import/importer/attachments/issues_importer.rb
+++ b/lib/gitlab/github_import/importer/attachments/issues_importer.rb
@@ -24,7 +24,7 @@ module Gitlab
private
def collection
- project.issues.select(:id, :description)
+ project.issues.select(:id, :description, :iid)
end
def ordering_column
diff --git a/lib/gitlab/github_import/importer/attachments/merge_requests_importer.rb b/lib/gitlab/github_import/importer/attachments/merge_requests_importer.rb
index f41071b1785..cd3a327a846 100644
--- a/lib/gitlab/github_import/importer/attachments/merge_requests_importer.rb
+++ b/lib/gitlab/github_import/importer/attachments/merge_requests_importer.rb
@@ -24,7 +24,7 @@ module Gitlab
private
def collection
- project.merge_requests.select(:id, :description)
+ project.merge_requests.select(:id, :description, :iid)
end
def ordering_column
diff --git a/lib/gitlab/github_import/importer/attachments/releases_importer.rb b/lib/gitlab/github_import/importer/attachments/releases_importer.rb
index feaa69eff71..7d6dbeb901e 100644
--- a/lib/gitlab/github_import/importer/attachments/releases_importer.rb
+++ b/lib/gitlab/github_import/importer/attachments/releases_importer.rb
@@ -24,7 +24,7 @@ module Gitlab
private
def collection
- project.releases.select(:id, :description)
+ project.releases.select(:id, :description, :tag)
end
end
end
diff --git a/lib/gitlab/github_import/importer/collaborator_importer.rb b/lib/gitlab/github_import/importer/collaborator_importer.rb
new file mode 100644
index 00000000000..9a90ea5a4ed
--- /dev/null
+++ b/lib/gitlab/github_import/importer/collaborator_importer.rb
@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module GithubImport
+ module Importer
+ class CollaboratorImporter
+ attr_reader :collaborator, :project, :client, :members_finder
+
+ # collaborator - An instance of `Gitlab::GithubImport::Representation::Collaborator`
+ # project - An instance of `Project`
+ # client - An instance of `Gitlab::GithubImport::Client`
+ def initialize(collaborator, project, client)
+ @collaborator = collaborator
+ @project = project
+ @client = client
+ @members_finder = ::MembersFinder.new(project, project.creator)
+ end
+
+ def execute
+ user_finder = GithubImport::UserFinder.new(project, client)
+ user_id = user_finder.user_id_for(collaborator)
+ return if user_id.nil?
+
+ membership = existing_user_membership(user_id)
+ access_level = map_access_level
+ return if membership && membership[:access_level] >= map_access_level
+
+ create_membership!(user_id, access_level)
+ end
+
+ private
+
+ def existing_user_membership(user_id)
+ members_finder.execute.find_by_user_id(user_id)
+ end
+
+ def map_access_level
+ access_level =
+ case collaborator[:role_name]
+ when 'read' then Gitlab::Access::GUEST
+ when 'triage' then Gitlab::Access::REPORTER
+ when 'write' then Gitlab::Access::DEVELOPER
+ when 'maintain' then Gitlab::Access::MAINTAINER
+ when 'admin' then Gitlab::Access::OWNER
+ end
+ return access_level if access_level
+
+ raise(
+ ::Gitlab::GithubImport::ObjectImporter::NotRetriableError,
+ "Unknown GitHub role: #{collaborator[:role_name]}"
+ )
+ end
+
+ def create_membership!(user_id, access_level)
+ ::ProjectMember.create!(
+ source: project,
+ access_level: access_level,
+ user_id: user_id,
+ member_namespace_id: project.project_namespace_id,
+ created_by_id: project.creator_id
+ )
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/github_import/importer/collaborators_importer.rb b/lib/gitlab/github_import/importer/collaborators_importer.rb
new file mode 100644
index 00000000000..7b18d3dba2a
--- /dev/null
+++ b/lib/gitlab/github_import/importer/collaborators_importer.rb
@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module GithubImport
+ module Importer
+ class CollaboratorsImporter
+ include ParallelScheduling
+
+ # The method that will be called for traversing through all the objects to
+ # import, yielding them to the supplied block.
+ def each_object_to_import
+ repo = project.import_source
+
+ direct_collaborators = client.collaborators(repo, affiliation: 'direct')
+ outside_collaborators = client.collaborators(repo, affiliation: 'outside')
+ collaborators_to_import = direct_collaborators.to_a - outside_collaborators.to_a
+
+ collaborators_to_import.each do |collaborator|
+ next if already_imported?(collaborator)
+
+ yield collaborator
+
+ Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched)
+ mark_as_imported(collaborator)
+ end
+ end
+
+ def importer_class
+ CollaboratorImporter
+ end
+
+ def representation_class
+ Representation::Collaborator
+ end
+
+ def sidekiq_worker_class
+ ImportCollaboratorWorker
+ end
+
+ def object_type
+ :collaborator
+ end
+
+ def collection_method
+ :collaborators
+ end
+
+ def id_for_already_imported_cache(collaborator)
+ collaborator[:id]
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/github_import/importer/events/cross_referenced.rb b/lib/gitlab/github_import/importer/events/cross_referenced.rb
index b56ae186d3c..4fe371e5900 100644
--- a/lib/gitlab/github_import/importer/events/cross_referenced.rb
+++ b/lib/gitlab/github_import/importer/events/cross_referenced.rb
@@ -55,6 +55,7 @@ module Gitlab
record = record_class.new(id: db_id, iid: iid)
record.project = project
+ record.namespace = project.project_namespace if record.respond_to?(:namespace)
record.readonly!
record
end
diff --git a/lib/gitlab/github_import/importer/label_links_importer.rb b/lib/gitlab/github_import/importer/label_links_importer.rb
index 52c87dda347..a20fec4b2ba 100644
--- a/lib/gitlab/github_import/importer/label_links_importer.rb
+++ b/lib/gitlab/github_import/importer/label_links_importer.rb
@@ -25,6 +25,8 @@ module Gitlab
items = []
target_id = find_target_id
+ return if target_id.blank?
+
issue.label_names.each do |label_name|
# Although unlikely it's technically possible for an issue to be
# given a label that was created and assigned after we imported all
diff --git a/lib/gitlab/github_import/importer/labels_importer.rb b/lib/gitlab/github_import/importer/labels_importer.rb
index d5d1cd28b7c..4554b932520 100644
--- a/lib/gitlab/github_import/importer/labels_importer.rb
+++ b/lib/gitlab/github_import/importer/labels_importer.rb
@@ -53,9 +53,18 @@ module Gitlab
:label
end
+ private
+
def model
Label
end
+
+ def github_identifiers(label)
+ {
+ title: label[:name],
+ object_type: object_type
+ }
+ end
end
end
end
diff --git a/lib/gitlab/github_import/importer/milestones_importer.rb b/lib/gitlab/github_import/importer/milestones_importer.rb
index 560fbdc66e3..cd6d450f15b 100644
--- a/lib/gitlab/github_import/importer/milestones_importer.rb
+++ b/lib/gitlab/github_import/importer/milestones_importer.rb
@@ -57,9 +57,19 @@ module Gitlab
:milestone
end
+ private
+
def model
Milestone
end
+
+ def github_identifiers(milestone)
+ {
+ iid: milestone[:number],
+ title: milestone[:title],
+ object_type: object_type
+ }
+ end
end
end
end
diff --git a/lib/gitlab/github_import/importer/note_attachments_importer.rb b/lib/gitlab/github_import/importer/note_attachments_importer.rb
index 9901c9e76f5..266ee2938ba 100644
--- a/lib/gitlab/github_import/importer/note_attachments_importer.rb
+++ b/lib/gitlab/github_import/importer/note_attachments_importer.rb
@@ -6,7 +6,7 @@ module Gitlab
class NoteAttachmentsImporter
attr_reader :note_text, :project
- # note_text - An instance of `NoteText`.
+ # note_text - An instance of `Gitlab::GithubImport::Representation::NoteText`.
# project - An instance of `Project`.
# client - An instance of `Gitlab::GithubImport::Client`.
def initialize(note_text, project, _client = nil)
@@ -19,7 +19,7 @@ module Gitlab
return if attachments.blank?
new_text = attachments.reduce(note_text.text) do |text, attachment|
- new_url = download_attachment(attachment)
+ new_url = gitlab_attachment_link(attachment)
text.gsub(attachment.url, new_url)
end
@@ -28,6 +28,28 @@ module Gitlab
private
+ def gitlab_attachment_link(attachment)
+ project_import_source = project.import_source
+
+ if attachment.part_of_project_blob?(project_import_source)
+ convert_project_content_link(attachment.url, project_import_source)
+ elsif attachment.media? || attachment.doc_belongs_to_project?(project_import_source)
+ download_attachment(attachment)
+ else # url to other GitHub project
+ attachment.url
+ end
+ end
+
+ # From: https://github.com/login/test-import-attachments-source/blob/main/example.md
+ # To: https://gitlab.com/login/test-import-attachments-target/-/blob/main/example.md
+ def convert_project_content_link(attachment_url, import_source)
+ path_without_domain = attachment_url.gsub(::Gitlab::GithubImport::MarkdownText.github_url, '')
+ path_without_import_source = path_without_domain.gsub(import_source, '').delete_prefix('/')
+ path_with_blob_prefix = "/-#{path_without_import_source}"
+
+ ::Gitlab::Routing.url_helpers.project_url(project) + path_with_blob_prefix
+ end
+
# in: an instance of Gitlab::GithubImport::Markdown::Attachment
# out: gitlab attachment markdown url
def download_attachment(attachment)
diff --git a/lib/gitlab/github_import/importer/pull_request_merged_by_importer.rb b/lib/gitlab/github_import/importer/pull_request_merged_by_importer.rb
deleted file mode 100644
index f05aa26a449..00000000000
--- a/lib/gitlab/github_import/importer/pull_request_merged_by_importer.rb
+++ /dev/null
@@ -1,73 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module GithubImport
- module Importer
- class PullRequestMergedByImporter
- # pull_request - An instance of
- # `Gitlab::GithubImport::Representation::PullRequest`
- # project - An instance of `Project`
- # client - An instance of `Gitlab::GithubImport::Client`
- def initialize(pull_request, project, client)
- @pull_request = pull_request
- @project = project
- @client = client
- end
-
- def execute
- user_finder = GithubImport::UserFinder.new(project, client)
-
- gitlab_user_id = begin
- user_finder.user_id_for(pull_request.merged_by)
- rescue ::Octokit::NotFound
- nil
- end
-
- metrics_upsert(gitlab_user_id)
-
- add_note!
- end
-
- private
-
- attr_reader :project, :pull_request, :client
-
- def metrics_upsert(gitlab_user_id)
- MergeRequest::Metrics.upsert({
- target_project_id: project.id,
- merge_request_id: merge_request.id,
- merged_by_id: gitlab_user_id,
- merged_at: pull_request.merged_at,
- created_at: timestamp,
- updated_at: timestamp
- }, unique_by: :merge_request_id)
- end
-
- def add_note!
- merge_request.notes.create!(
- importing: true,
- note: missing_author_note,
- author_id: project.creator_id,
- project: project,
- created_at: pull_request.merged_at
- )
- end
-
- def merge_request
- @merge_request ||= project.merge_requests.find_by_iid(pull_request.iid)
- end
-
- def timestamp
- @timestamp ||= Time.new.utc
- end
-
- def missing_author_note
- s_("GitHubImporter|*Merged by: %{author} at %{timestamp}*") % {
- author: pull_request.merged_by&.login || 'ghost',
- timestamp: pull_request.merged_at
- }
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/github_import/importer/pull_request_review_importer.rb b/lib/gitlab/github_import/importer/pull_request_review_importer.rb
deleted file mode 100644
index b1e259fe940..00000000000
--- a/lib/gitlab/github_import/importer/pull_request_review_importer.rb
+++ /dev/null
@@ -1,145 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module GithubImport
- module Importer
- class PullRequestReviewImporter
- # review - An instance of `Gitlab::GithubImport::Representation::PullRequestReview`
- # project - An instance of `Project`
- # client - An instance of `Gitlab::GithubImport::Client`
- def initialize(review, project, client)
- @review = review
- @project = project
- @client = client
- @merge_request = project.merge_requests.find_by_id(review.merge_request_id)
- end
-
- def execute
- user_finder = GithubImport::UserFinder.new(project, client)
-
- gitlab_user_id = begin
- user_finder.user_id_for(review.author)
- rescue ::Octokit::NotFound
- nil
- end
-
- if gitlab_user_id
- add_review_note!(gitlab_user_id)
- add_approval!(gitlab_user_id)
- add_reviewer!(gitlab_user_id)
- else
- add_complementary_review_note!(project.creator_id)
- end
- end
-
- private
-
- attr_reader :review, :merge_request, :project, :client
-
- def add_review_note!(author_id)
- return if review.note.empty?
-
- add_note!(author_id, review_note_content)
- end
-
- def add_complementary_review_note!(author_id)
- return if review.note.empty? && !review.approval?
-
- note_body = MarkdownText.format(
- review_note_content,
- review.author
- )
-
- add_note!(author_id, note_body)
- end
-
- def review_note_content
- header = "**Review:** #{review.review_type.humanize}"
-
- if review.note.present?
- "#{header}\n\n#{review.note}"
- else
- header
- end
- end
-
- def add_note!(author_id, note)
- note = Note.new(note_attributes(author_id, note))
-
- note.save!
- end
-
- def note_attributes(author_id, note, extra = {})
- {
- importing: true,
- noteable_id: merge_request.id,
- noteable_type: 'MergeRequest',
- project_id: project.id,
- author_id: author_id,
- note: note,
- system: false,
- created_at: submitted_at,
- updated_at: submitted_at
- }.merge(extra)
- end
-
- def add_approval!(user_id)
- return unless review.review_type == 'APPROVED'
-
- approval_attribues = {
- merge_request_id: merge_request.id,
- user_id: user_id,
- created_at: submitted_at,
- updated_at: submitted_at
- }
-
- result = ::Approval.insert(
- approval_attribues,
- returning: [:id],
- unique_by: [:user_id, :merge_request_id]
- )
-
- if result.rows.present?
- add_approval_system_note!(user_id)
- end
- end
-
- def add_reviewer!(user_id)
- return if review_re_requested?(user_id)
-
- ::MergeRequestReviewer.create!(
- merge_request_id: merge_request.id,
- user_id: user_id,
- state: ::MergeRequestReviewer.states['reviewed'],
- created_at: submitted_at
- )
- rescue ActiveRecord::RecordNotUnique
- # multiple reviews from single person could make a SQL concurrency issue here
- nil
- end
-
- # rubocop:disable CodeReuse/ActiveRecord
- def review_re_requested?(user_id)
- # records that were imported on previous stage with "unreviewed" status
- MergeRequestReviewer.where(merge_request_id: merge_request.id, user_id: user_id).exists?
- end
- # rubocop:enable CodeReuse/ActiveRecord
-
- def add_approval_system_note!(user_id)
- attributes = note_attributes(
- user_id,
- 'approved this merge request',
- system: true,
- system_note_metadata: SystemNoteMetadata.new(action: 'approved')
- )
-
- Note.create!(attributes)
- end
-
- def submitted_at
- @submitted_at ||= (review.submitted_at || merge_request.updated_at)
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/github_import/importer/pull_requests/all_merged_by_importer.rb b/lib/gitlab/github_import/importer/pull_requests/all_merged_by_importer.rb
new file mode 100644
index 00000000000..9aa55fd3eae
--- /dev/null
+++ b/lib/gitlab/github_import/importer/pull_requests/all_merged_by_importer.rb
@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module GithubImport
+ module Importer
+ module PullRequests
+ class AllMergedByImporter
+ include ParallelScheduling
+
+ def importer_class
+ MergedByImporter
+ end
+
+ def representation_class
+ Gitlab::GithubImport::Representation::PullRequest
+ end
+
+ def sidekiq_worker_class
+ Gitlab::GithubImport::PullRequests::ImportMergedByWorker
+ end
+
+ def collection_method
+ :pull_requests_merged_by
+ end
+
+ def object_type
+ :pull_request_merged_by
+ end
+
+ def id_for_already_imported_cache(merge_request)
+ merge_request.id
+ end
+
+ def each_object_to_import
+ merge_requests_to_import.find_each do |merge_request|
+ Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched)
+
+ pull_request = client.pull_request(project.import_source, merge_request.iid)
+ yield(pull_request)
+
+ mark_as_imported(merge_request)
+ end
+ end
+
+ private
+
+ # Returns only the merge requests that still have merged_by to be imported.
+ def merge_requests_to_import
+ project.merge_requests.id_not_in(already_imported_objects).with_state(:merged)
+ end
+
+ def already_imported_objects
+ Gitlab::Cache::Import::Caching.values_from_set(already_imported_cache_key)
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/github_import/importer/pull_requests/merged_by_importer.rb b/lib/gitlab/github_import/importer/pull_requests/merged_by_importer.rb
new file mode 100644
index 00000000000..19880716832
--- /dev/null
+++ b/lib/gitlab/github_import/importer/pull_requests/merged_by_importer.rb
@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module GithubImport
+ module Importer
+ module PullRequests
+ class MergedByImporter
+ # pull_request - An instance of
+ # `Gitlab::GithubImport::Representation::PullRequest`
+ # project - An instance of `Project`
+ # client - An instance of `Gitlab::GithubImport::Client`
+ def initialize(pull_request, project, client)
+ @pull_request = pull_request
+ @project = project
+ @client = client
+ end
+
+ def execute
+ user_finder = GithubImport::UserFinder.new(project, client)
+
+ gitlab_user_id = user_finder.user_id_for(pull_request.merged_by)
+
+ metrics_upsert(gitlab_user_id)
+
+ add_note!
+ end
+
+ private
+
+ attr_reader :project, :pull_request, :client
+
+ def metrics_upsert(gitlab_user_id)
+ MergeRequest::Metrics.upsert({
+ target_project_id: project.id,
+ merge_request_id: merge_request.id,
+ merged_by_id: gitlab_user_id,
+ merged_at: pull_request.merged_at,
+ created_at: timestamp,
+ updated_at: timestamp
+ }, unique_by: :merge_request_id)
+ end
+
+ def add_note!
+ merge_request.notes.create!(
+ importing: true,
+ note: missing_author_note,
+ author_id: project.creator_id,
+ project: project,
+ created_at: pull_request.merged_at
+ )
+ end
+
+ def merge_request
+ @merge_request ||= project.merge_requests.find_by_iid(pull_request.iid)
+ end
+
+ def timestamp
+ @timestamp ||= Time.new.utc
+ end
+
+ def missing_author_note
+ format(s_("GitHubImporter|*Merged by: %{author} at %{timestamp}*"),
+ author: pull_request.merged_by&.login || 'ghost',
+ timestamp: pull_request.merged_at
+ )
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/github_import/importer/pull_requests/review_importer.rb b/lib/gitlab/github_import/importer/pull_requests/review_importer.rb
new file mode 100644
index 00000000000..b250a42a53c
--- /dev/null
+++ b/lib/gitlab/github_import/importer/pull_requests/review_importer.rb
@@ -0,0 +1,141 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module GithubImport
+ module Importer
+ module PullRequests
+ class ReviewImporter
+ # review - An instance of `Gitlab::GithubImport::Representation::PullRequestReview`
+ # project - An instance of `Project`
+ # client - An instance of `Gitlab::GithubImport::Client`
+ def initialize(review, project, client)
+ @review = review
+ @project = project
+ @client = client
+ @merge_request = project.merge_requests.find_by_id(review.merge_request_id)
+ end
+
+ def execute
+ user_finder = GithubImport::UserFinder.new(project, client)
+
+ gitlab_user_id = user_finder.user_id_for(review.author)
+
+ if gitlab_user_id
+ add_review_note!(gitlab_user_id)
+ add_approval!(gitlab_user_id)
+ add_reviewer!(gitlab_user_id)
+ else
+ add_complementary_review_note!(project.creator_id)
+ end
+ end
+
+ private
+
+ attr_reader :review, :merge_request, :project, :client
+
+ def add_review_note!(author_id)
+ return if review.note.empty?
+
+ add_note!(author_id, review_note_content)
+ end
+
+ def add_complementary_review_note!(author_id)
+ return if review.note.empty? && !review.approval?
+
+ note_body = MarkdownText.format(
+ review_note_content,
+ review.author
+ )
+
+ add_note!(author_id, note_body)
+ end
+
+ def review_note_content
+ header = "**Review:** #{review.review_type.humanize}"
+
+ if review.note.present?
+ "#{header}\n\n#{review.note}"
+ else
+ header
+ end
+ end
+
+ def add_note!(author_id, note)
+ note = Note.new(note_attributes(author_id, note))
+
+ note.save!
+ end
+
+ def note_attributes(author_id, note, extra = {})
+ {
+ importing: true,
+ noteable_id: merge_request.id,
+ noteable_type: 'MergeRequest',
+ project_id: project.id,
+ author_id: author_id,
+ note: note,
+ system: false,
+ created_at: submitted_at,
+ updated_at: submitted_at
+ }.merge(extra)
+ end
+
+ def add_approval!(user_id)
+ return unless review.review_type == 'APPROVED'
+
+ approval_attribues = {
+ merge_request_id: merge_request.id,
+ user_id: user_id,
+ created_at: submitted_at,
+ updated_at: submitted_at
+ }
+
+ result = ::Approval.insert(
+ approval_attribues,
+ returning: [:id],
+ unique_by: [:user_id, :merge_request_id]
+ )
+
+ add_approval_system_note!(user_id) if result.rows.present?
+ end
+
+ def add_reviewer!(user_id)
+ return if review_re_requested?(user_id)
+
+ ::MergeRequestReviewer.create!(
+ merge_request_id: merge_request.id,
+ user_id: user_id,
+ state: ::MergeRequestReviewer.states['reviewed'],
+ created_at: submitted_at
+ )
+ rescue ActiveRecord::RecordNotUnique
+ # multiple reviews from single person could make a SQL concurrency issue here
+ nil
+ end
+
+ # rubocop:disable CodeReuse/ActiveRecord
+ def review_re_requested?(user_id)
+ # records that were imported on previous stage with "unreviewed" status
+ MergeRequestReviewer.where(merge_request_id: merge_request.id, user_id: user_id).exists?
+ end
+ # rubocop:enable CodeReuse/ActiveRecord
+
+ def add_approval_system_note!(user_id)
+ attributes = note_attributes(
+ user_id,
+ 'approved this merge request',
+ system: true,
+ system_note_metadata: SystemNoteMetadata.new(action: 'approved')
+ )
+
+ Note.create!(attributes)
+ end
+
+ def submitted_at
+ @submitted_at ||= (review.submitted_at || merge_request.updated_at)
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/github_import/importer/pull_requests/review_request_importer.rb b/lib/gitlab/github_import/importer/pull_requests/review_request_importer.rb
index bb51d856d9b..f51c610f24b 100644
--- a/lib/gitlab/github_import/importer/pull_requests/review_request_importer.rb
+++ b/lib/gitlab/github_import/importer/pull_requests/review_request_importer.rb
@@ -8,7 +8,6 @@ module Gitlab
def initialize(review_request, project, client)
@review_request = review_request
@user_finder = UserFinder.new(project, client)
- @issue_finder = IssuableFinder.new(project, client)
end
def execute
@@ -20,7 +19,7 @@ module Gitlab
attr_reader :review_request, :user_finder
def build_reviewers
- reviewer_ids = review_request.users.map { |user| user_finder.user_id_for(user) }.compact
+ reviewer_ids = review_request.users.filter_map { |user| user_finder.user_id_for(user) }
reviewer_ids.map do |reviewer_id|
MergeRequestReviewer.new(
diff --git a/lib/gitlab/github_import/importer/pull_requests/review_requests_importer.rb b/lib/gitlab/github_import/importer/pull_requests/review_requests_importer.rb
index c5d8da3be1c..0a92aee801d 100644
--- a/lib/gitlab/github_import/importer/pull_requests/review_requests_importer.rb
+++ b/lib/gitlab/github_import/importer/pull_requests/review_requests_importer.rb
@@ -18,6 +18,7 @@ module Gitlab
review_requests = client.pull_request_review_requests(repo, merge_request.iid)
review_requests[:merge_request_id] = merge_request.id
+ review_requests[:merge_request_iid] = merge_request.iid
yield review_requests
mark_merge_request_imported(merge_request)
diff --git a/lib/gitlab/github_import/importer/pull_requests/reviews_importer.rb b/lib/gitlab/github_import/importer/pull_requests/reviews_importer.rb
new file mode 100644
index 00000000000..347423b0e21
--- /dev/null
+++ b/lib/gitlab/github_import/importer/pull_requests/reviews_importer.rb
@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module GithubImport
+ module Importer
+ module PullRequests
+ class ReviewsImporter
+ include ParallelScheduling
+
+ def initialize(...)
+ super
+
+ @merge_requests_already_imported_cache_key =
+ "github-importer/merge_request/already-imported/#{project.id}"
+ end
+
+ def importer_class
+ ReviewImporter
+ end
+
+ def representation_class
+ Gitlab::GithubImport::Representation::PullRequestReview
+ end
+
+ def sidekiq_worker_class
+ Gitlab::GithubImport::PullRequests::ImportReviewWorker
+ end
+
+ def collection_method
+ :pull_request_reviews
+ end
+
+ def object_type
+ :pull_request_review
+ end
+
+ def id_for_already_imported_cache(review)
+ review[:id]
+ end
+
+ # The worker can be interrupted, by rate limit for instance,
+ # in different situations. To avoid requesting already imported data,
+ # if the worker is interrupted:
+ # - before importing all reviews of a merge request
+ # The reviews page is cached with the `PageCounter`, by merge request.
+ # - before importing all merge requests reviews
+ # Merge requests that had all the reviews imported are cached with
+ # `mark_merge_request_reviews_imported`
+ def each_object_to_import(&_block)
+ each_review_page do |page, merge_request|
+ page.objects.each do |review|
+ review = review.to_h
+
+ next if already_imported?(review)
+
+ Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched)
+
+ review[:merge_request_id] = merge_request.id
+ review[:merge_request_iid] = merge_request.iid
+ yield(review)
+
+ mark_as_imported(review)
+ end
+ end
+ end
+
+ private
+
+ attr_reader :merge_requests_already_imported_cache_key
+
+ def each_review_page
+ merge_requests_to_import.find_each do |merge_request|
+ # The page counter needs to be scoped by merge request to avoid skipping
+ # pages of reviews from already imported merge requests.
+ page_counter = PageCounter.new(project, page_counter_id(merge_request))
+ repo = project.import_source
+ options = collection_options.merge(page: page_counter.current)
+
+ client.each_page(collection_method, repo, merge_request.iid, options) do |page|
+ next unless page_counter.set(page.number)
+
+ yield(page, merge_request)
+ end
+
+ # Avoid unnecessary Redis cache keys after the work is done.
+ page_counter.expire!
+ mark_merge_request_reviews_imported(merge_request)
+ end
+ end
+
+ # Returns only the merge requests that still have reviews to be imported.
+ def merge_requests_to_import
+ project.merge_requests.id_not_in(already_imported_merge_requests)
+ end
+
+ def already_imported_merge_requests
+ Gitlab::Cache::Import::Caching.values_from_set(merge_requests_already_imported_cache_key)
+ end
+
+ def page_counter_id(merge_request)
+ "merge_request/#{merge_request.id}/#{collection_method}"
+ end
+
+ def mark_merge_request_reviews_imported(merge_request)
+ Gitlab::Cache::Import::Caching.set_add(
+ merge_requests_already_imported_cache_key,
+ merge_request.id
+ )
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/github_import/importer/pull_requests_merged_by_importer.rb b/lib/gitlab/github_import/importer/pull_requests_merged_by_importer.rb
deleted file mode 100644
index c56b391cbec..00000000000
--- a/lib/gitlab/github_import/importer/pull_requests_merged_by_importer.rb
+++ /dev/null
@@ -1,57 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module GithubImport
- module Importer
- class PullRequestsMergedByImporter
- include ParallelScheduling
-
- def importer_class
- PullRequestMergedByImporter
- end
-
- def representation_class
- Gitlab::GithubImport::Representation::PullRequest
- end
-
- def sidekiq_worker_class
- ImportPullRequestMergedByWorker
- end
-
- def collection_method
- :pull_requests_merged_by
- end
-
- def object_type
- :pull_request_merged_by
- end
-
- def id_for_already_imported_cache(merge_request)
- merge_request.id
- end
-
- def each_object_to_import
- merge_requests_to_import.find_each do |merge_request|
- Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched)
-
- pull_request = client.pull_request(project.import_source, merge_request.iid)
- yield(pull_request)
-
- mark_as_imported(merge_request)
- end
- end
-
- private
-
- # Returns only the merge requests that still have merged_by to be imported.
- def merge_requests_to_import
- project.merge_requests.id_not_in(already_imported_objects).with_state(:merged)
- end
-
- def already_imported_objects
- Gitlab::Cache::Import::Caching.values_from_set(already_imported_cache_key)
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb b/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb
deleted file mode 100644
index 543c29a21a0..00000000000
--- a/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb
+++ /dev/null
@@ -1,111 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module GithubImport
- module Importer
- class PullRequestsReviewsImporter
- include ParallelScheduling
-
- def initialize(...)
- super
-
- @merge_requests_already_imported_cache_key =
- "github-importer/merge_request/already-imported/#{project.id}"
- end
-
- def importer_class
- PullRequestReviewImporter
- end
-
- def representation_class
- Gitlab::GithubImport::Representation::PullRequestReview
- end
-
- def sidekiq_worker_class
- ImportPullRequestReviewWorker
- end
-
- def collection_method
- :pull_request_reviews
- end
-
- def object_type
- :pull_request_review
- end
-
- def id_for_already_imported_cache(review)
- review[:id]
- end
-
- # The worker can be interrupted, by rate limit for instance,
- # in different situations. To avoid requesting already imported data,
- # if the worker is interrupted:
- # - before importing all reviews of a merge request
- # The reviews page is cached with the `PageCounter`, by merge request.
- # - before importing all merge requests reviews
- # Merge requests that had all the reviews imported are cached with
- # `mark_merge_request_reviews_imported`
- def each_object_to_import(&block)
- each_review_page do |page, merge_request|
- page.objects.each do |review|
- review = review.to_h
-
- next if already_imported?(review)
-
- Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched)
-
- review[:merge_request_id] = merge_request.id
- yield(review)
-
- mark_as_imported(review)
- end
- end
- end
-
- private
-
- attr_reader :merge_requests_already_imported_cache_key
-
- def each_review_page
- merge_requests_to_import.find_each do |merge_request|
- # The page counter needs to be scoped by merge request to avoid skipping
- # pages of reviews from already imported merge requests.
- page_counter = PageCounter.new(project, page_counter_id(merge_request))
- repo = project.import_source
- options = collection_options.merge(page: page_counter.current)
-
- client.each_page(collection_method, repo, merge_request.iid, options) do |page|
- next unless page_counter.set(page.number)
-
- yield(page, merge_request)
- end
-
- # Avoid unnecessary Redis cache keys after the work is done.
- page_counter.expire!
- mark_merge_request_reviews_imported(merge_request)
- end
- end
-
- # Returns only the merge requests that still have reviews to be imported.
- def merge_requests_to_import
- project.merge_requests.id_not_in(already_imported_merge_requests)
- end
-
- def already_imported_merge_requests
- Gitlab::Cache::Import::Caching.values_from_set(merge_requests_already_imported_cache_key)
- end
-
- def page_counter_id(merge_request)
- "merge_request/#{merge_request.id}/#{collection_method}"
- end
-
- def mark_merge_request_reviews_imported(merge_request)
- Gitlab::Cache::Import::Caching.set_add(
- merge_requests_already_imported_cache_key,
- merge_request.id
- )
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/github_import/importer/releases_importer.rb b/lib/gitlab/github_import/importer/releases_importer.rb
index 62d579fda08..2f210dafd0c 100644
--- a/lib/gitlab/github_import/importer/releases_importer.rb
+++ b/lib/gitlab/github_import/importer/releases_importer.rb
@@ -73,6 +73,13 @@ module Gitlab
def model
Release
end
+
+ def github_identifiers(release)
+ {
+ tag: release[:tag_name],
+ object_type: object_type
+ }
+ end
end
end
end
diff --git a/lib/gitlab/github_import/importer/repository_importer.rb b/lib/gitlab/github_import/importer/repository_importer.rb
index d7fe01e90f8..2654812b64a 100644
--- a/lib/gitlab/github_import/importer/repository_importer.rb
+++ b/lib/gitlab/github_import/importer/repository_importer.rb
@@ -66,13 +66,10 @@ module Gitlab
true
rescue ::Gitlab::Git::CommandError => e
- if e.message !~ /repository not exported/
- project.create_wiki
+ return true if e.message.include?('repository not exported')
- raise e
- else
- true
- end
+ project.create_wiki
+ raise e
end
def wiki_url
@@ -89,10 +86,8 @@ module Gitlab
client_repository[:default_branch]
end
- def client_repository
- strong_memoize(:client_repository) do
- client.repository(project.import_source)
- end
+ strong_memoize_attr def client_repository
+ client.repository(project.import_source)
end
end
end
diff --git a/lib/gitlab/github_import/job_delay_calculator.rb b/lib/gitlab/github_import/job_delay_calculator.rb
new file mode 100644
index 00000000000..52b211c92d6
--- /dev/null
+++ b/lib/gitlab/github_import/job_delay_calculator.rb
@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module GithubImport
+ # Used to calculate delay to spread sidekiq jobs on fetching records during import
+ # and upon job reschedule when the rate limit is reached
+ module JobDelayCalculator
+ # Default batch settings for parallel import (can be redefined in Importer/Worker classes)
+ def parallel_import_batch
+ { size: 1000, delay: 1.minute }
+ end
+
+ private
+
+ def calculate_job_delay(job_index)
+ multiplier = (job_index / parallel_import_batch[:size])
+
+ (multiplier * parallel_import_batch[:delay]) + 1.second
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/github_import/markdown/attachment.rb b/lib/gitlab/github_import/markdown/attachment.rb
index 1c814e34a39..e270cfba619 100644
--- a/lib/gitlab/github_import/markdown/attachment.rb
+++ b/lib/gitlab/github_import/markdown/attachment.rb
@@ -79,6 +79,22 @@ module Gitlab
@url = url
end
+ def part_of_project_blob?(import_source)
+ url.start_with?(
+ "#{::Gitlab::GithubImport::MarkdownText.github_url}/#{import_source}/blob"
+ )
+ end
+
+ def doc_belongs_to_project?(import_source)
+ url.start_with?(
+ "#{::Gitlab::GithubImport::MarkdownText.github_url}/#{import_source}/files"
+ )
+ end
+
+ def media?
+ url.start_with?(::Gitlab::GithubImport::MarkdownText::GITHUB_MEDIA_CDN)
+ end
+
def inspect
"<#{self.class.name}: { name: #{name}, url: #{url} }>"
end
diff --git a/lib/gitlab/github_import/parallel_scheduling.rb b/lib/gitlab/github_import/parallel_scheduling.rb
index 4b54a77983d..cfc1ec526b0 100644
--- a/lib/gitlab/github_import/parallel_scheduling.rb
+++ b/lib/gitlab/github_import/parallel_scheduling.rb
@@ -3,6 +3,8 @@
module Gitlab
module GithubImport
module ParallelScheduling
+ include JobDelayCalculator
+
attr_reader :project, :client, :page_counter, :already_imported_cache_key,
:job_waiter_cache_key, :job_waiter_remaining_cache_key
@@ -85,14 +87,10 @@ module Gitlab
def parallel_import
raise 'Batch settings must be defined for parallel import' if parallel_import_batch.blank?
- if Feature.enabled?(:improved_spread_parallel_import)
- improved_spread_parallel_import
- else
- spread_parallel_import
- end
+ spread_parallel_import
end
- def improved_spread_parallel_import
+ def spread_parallel_import
enqueued_job_counter = 0
each_object_to_import do |object|
@@ -108,33 +106,6 @@ module Gitlab
job_waiter
end
- def spread_parallel_import
- waiter = JobWaiter.new
-
- import_arguments = []
-
- each_object_to_import do |object|
- repr = object_representation(object)
-
- import_arguments << [project.id, repr.to_hash, waiter.key]
-
- waiter.jobs_remaining += 1
- end
-
- # rubocop:disable Scalability/BulkPerformWithContext
- Gitlab::ApplicationContext.with_context(project: project) do
- sidekiq_worker_class.bulk_perform_in(
- 1.second,
- import_arguments,
- batch_size: parallel_import_batch[:size],
- batch_delay: parallel_import_batch[:delay]
- )
- end
- # rubocop:enable Scalability/BulkPerformWithContext
-
- waiter
- end
-
# The method that will be called for traversing through all the objects to
# import, yielding them to the supplied block.
def each_object_to_import
@@ -228,11 +199,6 @@ module Gitlab
raise NotImplementedError
end
- # Default batch settings for parallel import (can be redefined in Importer classes)
- def parallel_import_batch
- { size: 1000, delay: 1.minute }
- end
-
def abort_on_failure
false
end
@@ -274,12 +240,6 @@ module Gitlab
JobWaiter.new(jobs_remaining, key)
end
end
-
- def calculate_job_delay(job_index)
- multiplier = (job_index / parallel_import_batch[:size])
-
- (multiplier * parallel_import_batch[:delay]) + 1.second
- end
end
end
end
diff --git a/lib/gitlab/github_import/project_relation_type.rb b/lib/gitlab/github_import/project_relation_type.rb
new file mode 100644
index 00000000000..a6e598172ee
--- /dev/null
+++ b/lib/gitlab/github_import/project_relation_type.rb
@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module GithubImport
+ class ProjectRelationType
+ CACHE_ORGS_EXPIRES_IN = 5.minutes
+ CACHE_USER_EXPIRES_IN = 1.hour
+
+ def initialize(client)
+ @client = client
+ end
+
+ def for(import_source)
+ namespace = import_source.split('/')[0]
+ if user?(namespace)
+ 'owned'
+ elsif organization?(namespace)
+ 'organization'
+ else
+ 'collaborated'
+ end
+ end
+
+ private
+
+ attr_reader :client
+
+ def user?(namespace)
+ github_user_login == namespace
+ end
+
+ def organization?(namespace)
+ github_org_logins.include? namespace
+ end
+
+ def github_user_login
+ ::Rails.cache.fetch(cache_key('user_login'), expire_in: CACHE_USER_EXPIRES_IN) do
+ client.user(nil)[:login]
+ end
+ end
+
+ def github_org_logins
+ ::Rails.cache.fetch(cache_key('organization_logins'), expires_in: CACHE_ORGS_EXPIRES_IN) do
+ logins = []
+ client.each_object(:organizations) { |org| logins.push(org[:login]) }
+ logins
+ end
+ end
+
+ def cache_key(subject)
+ ['github_import', Gitlab::CryptoHelper.sha256(client.octokit.access_token), subject].join('/')
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/github_import/representation/collaborator.rb b/lib/gitlab/github_import/representation/collaborator.rb
new file mode 100644
index 00000000000..fb58a572151
--- /dev/null
+++ b/lib/gitlab/github_import/representation/collaborator.rb
@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module GithubImport
+ module Representation
+ class Collaborator
+ include ToHash
+ include ExposeAttribute
+
+ attr_reader :attributes
+
+ expose_attribute :id, :login, :role_name
+
+ # Builds a user from a GitHub API response.
+ #
+ # collaborator - An instance of `Hash` containing the user & role details.
+ def self.from_api_response(collaborator, _additional_data = {})
+ new(
+ id: collaborator[:id],
+ login: collaborator[:login],
+ role_name: collaborator[:role_name]
+ )
+ end
+
+ # Builds a user using a Hash that was built from a JSON payload.
+ def self.from_json_hash(raw_hash)
+ new(Representation.symbolize_hash(raw_hash))
+ end
+
+ # attributes - A Hash containing the user details. The keys of this
+ # Hash (and any nested hashes) must be symbols.
+ def initialize(attributes)
+ @attributes = attributes
+ end
+
+ def github_identifiers
+ {
+ id: id,
+ login: login
+ }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/github_import/representation/diff_note.rb b/lib/gitlab/github_import/representation/diff_note.rb
index 9259d0295d5..0408b34bb02 100644
--- a/lib/gitlab/github_import/representation/diff_note.rb
+++ b/lib/gitlab/github_import/representation/diff_note.rb
@@ -120,7 +120,7 @@ module Gitlab
def github_identifiers
{
note_id: note_id,
- noteable_id: noteable_id,
+ noteable_iid: noteable_id,
noteable_type: noteable_type
}
end
diff --git a/lib/gitlab/github_import/representation/issue.rb b/lib/gitlab/github_import/representation/issue.rb
index e878aeaf3b9..95a7c5ebf4b 100644
--- a/lib/gitlab/github_import/representation/issue.rb
+++ b/lib/gitlab/github_import/representation/issue.rb
@@ -79,7 +79,8 @@ module Gitlab
def github_identifiers
{
iid: iid,
- issuable_type: issuable_type
+ issuable_type: issuable_type,
+ title: title
}
end
end
diff --git a/lib/gitlab/github_import/representation/issue_event.rb b/lib/gitlab/github_import/representation/issue_event.rb
index 39a23c016ce..068d5cf9482 100644
--- a/lib/gitlab/github_import/representation/issue_event.rb
+++ b/lib/gitlab/github_import/representation/issue_event.rb
@@ -20,7 +20,11 @@ module Gitlab
end
def github_identifiers
- { id: id }
+ {
+ id: id,
+ issuable_iid: issuable_id,
+ event: event
+ }
end
def issuable_type
diff --git a/lib/gitlab/github_import/representation/lfs_object.rb b/lib/gitlab/github_import/representation/lfs_object.rb
index cd614db2161..716e77bf401 100644
--- a/lib/gitlab/github_import/representation/lfs_object.rb
+++ b/lib/gitlab/github_import/representation/lfs_object.rb
@@ -33,7 +33,8 @@ module Gitlab
def github_identifiers
{
- oid: oid
+ oid: oid,
+ size: size
}
end
end
diff --git a/lib/gitlab/github_import/representation/note.rb b/lib/gitlab/github_import/representation/note.rb
index 14379e8a4e9..7a8bdfb1c64 100644
--- a/lib/gitlab/github_import/representation/note.rb
+++ b/lib/gitlab/github_import/representation/note.rb
@@ -76,7 +76,7 @@ module Gitlab
def github_identifiers
{
note_id: note_id,
- noteable_id: noteable_id,
+ noteable_iid: noteable_id,
noteable_type: noteable_type
}
end
diff --git a/lib/gitlab/github_import/representation/note_text.rb b/lib/gitlab/github_import/representation/note_text.rb
index 505d7d805d3..70dd242303a 100644
--- a/lib/gitlab/github_import/representation/note_text.rb
+++ b/lib/gitlab/github_import/representation/note_text.rb
@@ -16,35 +16,35 @@ module Gitlab
attr_reader :attributes
- expose_attribute :record_db_id, :record_type, :text
-
- class << self
- # Builds a note text representation from DB record of Note or Release.
- #
- # record - An instance of `Note`, `Release`, `Issue`, `MergeRequest` model
- def from_db_record(record)
- check_record_class!(record)
-
- record_type = record.class.name
- # only column for note is different along MODELS_ALLOWLIST
- text = record.is_a?(::Note) ? record.note : record.description
- new(
- record_db_id: record.id,
- record_type: record_type,
- text: text
- )
- end
+ expose_attribute :record_db_id, :record_type, :text, :iid, :tag, :noteable_type
- def from_json_hash(raw_hash)
- new Representation.symbolize_hash(raw_hash)
- end
+ # Builds a note text representation from DB record of Note or Release.
+ #
+ # record - An instance of `Note`, `Release`, `Issue`, `MergeRequest` model
+ def self.from_db_record(record)
+ check_record_class!(record)
- private
+ record_type = record.class.name
+ # only column for note is different along MODELS_ALLOWLIST
+ text = record.is_a?(::Note) ? record.note : record.description
+ new(
+ record_db_id: record.id,
+ record_type: record_type,
+ text: text,
+ iid: record.try(:iid),
+ tag: record.try(:tag),
+ noteable_type: record.try(:noteable_type)
+ )
+ end
- def check_record_class!(record)
- raise ModelNotSupported, record.class.name if MODELS_ALLOWLIST.exclude?(record.class)
- end
+ def self.from_json_hash(raw_hash)
+ new Representation.symbolize_hash(raw_hash)
+ end
+
+ def self.check_record_class!(record)
+ raise ModelNotSupported, record.class.name if MODELS_ALLOWLIST.exclude?(record.class)
end
+ private_class_method :check_record_class!
# attributes - A Hash containing the event details. The keys of this
# Hash (and any nested hashes) must be symbols.
@@ -53,7 +53,22 @@ module Gitlab
end
def github_identifiers
- { db_id: record_db_id }
+ {
+ db_id: record_db_id
+ }.merge(record_type_specific_attribute)
+ end
+
+ private
+
+ def record_type_specific_attribute
+ case record_type
+ when ::Release.name
+ { tag: tag }
+ when ::Issue.name, ::MergeRequest.name
+ { noteable_iid: iid }
+ when ::Note.name
+ { noteable_type: noteable_type }
+ end
end
end
end
diff --git a/lib/gitlab/github_import/representation/pull_request.rb b/lib/gitlab/github_import/representation/pull_request.rb
index 4b8ae1f8eab..f26fa953773 100644
--- a/lib/gitlab/github_import/representation/pull_request.rb
+++ b/lib/gitlab/github_import/representation/pull_request.rb
@@ -111,7 +111,8 @@ module Gitlab
def github_identifiers
{
iid: iid,
- issuable_type: issuable_type
+ issuable_type: issuable_type,
+ title: title
}
end
end
diff --git a/lib/gitlab/github_import/representation/pull_request_review.rb b/lib/gitlab/github_import/representation/pull_request_review.rb
index 8fb57ae89a4..0c6e281cd6d 100644
--- a/lib/gitlab/github_import/representation/pull_request_review.rb
+++ b/lib/gitlab/github_import/representation/pull_request_review.rb
@@ -9,7 +9,7 @@ module Gitlab
attr_reader :attributes
- expose_attribute :author, :note, :review_type, :submitted_at, :merge_request_id, :review_id
+ expose_attribute :author, :note, :review_type, :submitted_at, :merge_request_id, :merge_request_iid, :review_id
# Builds a PullRequestReview from a GitHub API response.
#
@@ -19,6 +19,7 @@ module Gitlab
new(
merge_request_id: review[:merge_request_id],
+ merge_request_iid: review[:merge_request_iid],
author: user,
note: review[:body],
review_type: review[:state],
@@ -49,8 +50,8 @@ module Gitlab
def github_identifiers
{
- review_id: review_id,
- merge_request_id: merge_request_id
+ merge_request_iid: merge_request_iid,
+ review_id: review_id
}
end
end
diff --git a/lib/gitlab/github_import/representation/pull_requests/review_requests.rb b/lib/gitlab/github_import/representation/pull_requests/review_requests.rb
index 692004c4460..a6ec1d3178b 100644
--- a/lib/gitlab/github_import/representation/pull_requests/review_requests.rb
+++ b/lib/gitlab/github_import/representation/pull_requests/review_requests.rb
@@ -10,7 +10,7 @@ module Gitlab
attr_reader :attributes
- expose_attribute :merge_request_id, :users
+ expose_attribute :merge_request_id, :merge_request_iid, :users
class << self
# Builds a list of requested reviewers from a GitHub API response.
@@ -24,6 +24,7 @@ module Gitlab
new(
merge_request_id: review_requests[:merge_request_id],
+ merge_request_iid: review_requests[:merge_request_iid],
users: users
)
end
@@ -37,7 +38,10 @@ module Gitlab
end
def github_identifiers
- { merge_request_id: merge_request_id }
+ {
+ merge_request_iid: merge_request_iid,
+ requested_reviewers: users.pluck(:login) # rubocop: disable CodeReuse/ActiveRecord
+ }
end
end
end
diff --git a/lib/gitlab/github_import/settings.rb b/lib/gitlab/github_import/settings.rb
index 77288b9fb98..0b883de8ed0 100644
--- a/lib/gitlab/github_import/settings.rb
+++ b/lib/gitlab/github_import/settings.rb
@@ -6,6 +6,7 @@ module Gitlab
OPTIONAL_STAGES = {
single_endpoint_issue_events_import: {
label: 'Import issue and pull request events',
+ selected: false,
details: <<-TEXT.split("\n").map(&:strip).join(' ')
For example, opened or closed, renamed, and labeled or unlabeled.
Time required to import these events depends on how many issues or pull requests your project has.
@@ -13,17 +14,27 @@ module Gitlab
},
single_endpoint_notes_import: {
label: 'Use alternative comments import method',
+ selected: false,
details: <<-TEXT.split("\n").map(&:strip).join(' ')
The default method can skip some comments in large projects because of limitations of the GitHub API.
TEXT
},
attachments_import: {
- label: 'Import Markdown attachments',
+ label: 'Import Markdown attachments (links)',
+ selected: false,
details: <<-TEXT.split("\n").map(&:strip).join(' ')
- Import Markdown attachments from repository comments, release posts, issue descriptions,
+ Import Markdown attachments (links) from repository comments, release posts, issue descriptions,
and pull request descriptions. These can include images, text, or binary attachments.
If not imported, links in Markdown to attachments break after you remove the attachments from GitHub.
TEXT
+ },
+ collaborators_import: {
+ label: 'Import collaborators',
+ selected: true,
+ details: <<-TEXT.split("\n").map(&:strip).join(' ')
+ Import direct repository collaborators who are not outside collaborators.
+ Imported collaborators who aren't members of the group you imported the project into consume seats on your GitLab instance.
+ TEXT
}
}.freeze
@@ -32,6 +43,7 @@ module Gitlab
{
name: stage_name.to_s,
label: s_(format("GitHubImport|%{text}", text: data[:label])),
+ selected: data[:selected],
details: s_(format("GitHubImport|%{text}", text: data[:details]))
}
end
diff --git a/lib/gitlab/github_import/user_finder.rb b/lib/gitlab/github_import/user_finder.rb
index b8751def08f..dd71edbd205 100644
--- a/lib/gitlab/github_import/user_finder.rb
+++ b/lib/gitlab/github_import/user_finder.rb
@@ -28,6 +28,9 @@ module Gitlab
EMAIL_FOR_USERNAME_CACHE_KEY =
'github-import/user-finder/email-for-username/%s'
+ # The base cache key to use for caching inexistence of GitHub usernames.
+ INEXISTENCE_OF_GITHUB_USERNAME_CACHE_KEY = 'github-import/user-finder/inexistence-of-username/%s'
+
# project - An instance of `Project`
# client - An instance of `Gitlab::GithubImport::Client`
def initialize(project, client)
@@ -113,12 +116,15 @@ module Gitlab
cache_key = EMAIL_FOR_USERNAME_CACHE_KEY % username
email = Gitlab::Cache::Import::Caching.read(cache_key)
- unless email
+ if email.blank? && !github_username_inexists?(username)
user = client.user(username)
email = Gitlab::Cache::Import::Caching.write(cache_key, user[:email], timeout: timeout(user[:email])) if user
end
email
+ rescue ::Octokit::NotFound
+ cache_github_username_inexistence(username)
+ nil
end
def cached_id_for_github_id(id)
@@ -190,6 +196,18 @@ module Gitlab
Gitlab::Cache::Import::Caching::SHORTER_TIMEOUT
end
end
+
+ def github_username_inexists?(username)
+ cache_key = INEXISTENCE_OF_GITHUB_USERNAME_CACHE_KEY % username
+
+ Gitlab::Cache::Import::Caching.read(cache_key) == 'true'
+ end
+
+ def cache_github_username_inexistence(username)
+ cache_key = INEXISTENCE_OF_GITHUB_USERNAME_CACHE_KEY % username
+
+ Gitlab::Cache::Import::Caching.write(cache_key, true)
+ end
end
end
end
diff --git a/lib/gitlab/gitlab_import/client.rb b/lib/gitlab/gitlab_import/client.rb
deleted file mode 100644
index 86474159f8b..00000000000
--- a/lib/gitlab/gitlab_import/client.rb
+++ /dev/null
@@ -1,89 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module GitlabImport
- class Client
- attr_reader :client, :api
-
- PER_PAGE = 100
-
- def initialize(access_token)
- @client = ::OAuth2::Client.new(
- config.app_id,
- config.app_secret,
- gitlab_options
- )
-
- if access_token
- @api = OAuth2::AccessToken.from_hash(@client, access_token: access_token)
- end
- end
-
- def authorize_url(redirect_uri)
- client.auth_code.authorize_url({
- redirect_uri: redirect_uri,
- scope: "api"
- })
- end
-
- def get_token(code, redirect_uri)
- client.auth_code.get_token(code, redirect_uri: redirect_uri).token
- end
-
- def user
- api.get("/api/v4/user").parsed
- end
-
- def issues(project_identifier, **kwargs)
- lazy_page_iterator(**kwargs) do |page, per_page|
- api.get("/api/v4/projects/#{project_identifier}/issues?per_page=#{per_page}&page=#{page}").parsed
- end
- end
-
- def issue_comments(project_identifier, issue_id, **kwargs)
- lazy_page_iterator(**kwargs) do |page, per_page|
- api.get("/api/v4/projects/#{project_identifier}/issues/#{issue_id}/notes?per_page=#{per_page}&page=#{page}").parsed
- end
- end
-
- def project(id)
- api.get("/api/v4/projects/#{id}").parsed
- end
-
- def projects(**kwargs)
- lazy_page_iterator(**kwargs) do |page, per_page|
- api.get("/api/v4/projects?per_page=#{per_page}&page=#{page}&simple=true&membership=true").parsed
- end
- end
-
- private
-
- def lazy_page_iterator(starting_page: 1, page_limit: nil, per_page: PER_PAGE)
- Enumerator.new do |y|
- page = starting_page
- page_limit = (starting_page - 1) + page_limit if page_limit
-
- loop do
- items = yield(page, per_page)
-
- items.each do |item|
- y << item
- end
-
- break if items.empty? || items.size < per_page || (page_limit && page >= page_limit)
-
- page += 1
- end
- end
- end
-
- def config
- Gitlab::Auth::OAuth::Provider.config_for('gitlab')
- end
-
- def gitlab_options
- OmniAuth::Strategies::GitLab.default_options[:client_options].to_h.symbolize_keys
- end
- end
- end
-end
diff --git a/lib/gitlab/gitlab_import/importer.rb b/lib/gitlab/gitlab_import/importer.rb
deleted file mode 100644
index 0cd33b9f5b7..00000000000
--- a/lib/gitlab/gitlab_import/importer.rb
+++ /dev/null
@@ -1,65 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module GitlabImport
- class Importer
- attr_reader :project, :client
-
- def initialize(project)
- @project = project
- import_data = project.import_data
- if import_data && import_data.credentials && import_data.credentials[:password]
- @client = Client.new(import_data.credentials[:password])
- @formatter = Gitlab::ImportFormatter.new
- else
- raise Projects::ImportService::Error, "Unable to find project import data credentials for project ID: #{@project.id}"
- end
- end
-
- def execute
- ActiveRecord::Base.no_touching do
- project_identifier = CGI.escape(project.import_source)
-
- # Issues && Comments
- issues = client.issues(project_identifier)
-
- issues.each do |issue|
- body = [@formatter.author_line(issue["author"]["name"])]
- body << issue["description"]
-
- comments = client.issue_comments(project_identifier, issue["iid"])
-
- if comments.any?
- body << @formatter.comments_header
- end
-
- comments.each do |comment|
- body << @formatter.comment(comment["author"]["name"], comment["created_at"], comment["body"])
- end
-
- project.issues.create!(
- iid: issue["iid"],
- description: body.join,
- title: issue["title"],
- state: issue["state"],
- updated_at: issue["updated_at"],
- author_id: gitlab_user_id(project, issue["author"]["id"]),
- confidential: issue["confidential"]
- )
- end
- end
-
- true
- end
-
- private
-
- # rubocop: disable CodeReuse/ActiveRecord
- def gitlab_user_id(project, gitlab_id)
- user_id = User.by_provider_and_extern_uid(:gitlab, gitlab_id).select(:id).first&.id
- user_id || project.creator_id
- end
- # rubocop: enable CodeReuse/ActiveRecord
- end
- end
-end
diff --git a/lib/gitlab/gitlab_import/project_creator.rb b/lib/gitlab/gitlab_import/project_creator.rb
deleted file mode 100644
index 35feea17351..00000000000
--- a/lib/gitlab/gitlab_import/project_creator.rb
+++ /dev/null
@@ -1,30 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module GitlabImport
- class ProjectCreator
- attr_reader :repo, :namespace, :current_user, :session_data
-
- def initialize(repo, namespace, current_user, session_data)
- @repo = repo
- @namespace = namespace
- @current_user = current_user
- @session_data = session_data
- end
-
- def execute
- ::Projects::CreateService.new(
- current_user,
- name: repo["name"],
- path: repo["path"],
- description: repo["description"],
- namespace_id: namespace.id,
- visibility_level: Gitlab::VisibilityLevel.level_value(repo["visibility"]),
- import_type: "gitlab",
- import_source: repo["path_with_namespace"],
- import_url: repo["http_url_to_repo"].sub("://", "://oauth2:#{@session_data[:gitlab_access_token]}@")
- ).execute
- end
- end
- end
-end
diff --git a/lib/gitlab/gl_repository.rb b/lib/gitlab/gl_repository.rb
index d123989ef8e..268d5d3e564 100644
--- a/lib/gitlab/gl_repository.rb
+++ b/lib/gitlab/gl_repository.rb
@@ -34,8 +34,9 @@ module Gitlab
DESIGN = ::Gitlab::GlRepository::RepoType.new(
name: :design,
access_checker_class: ::Gitlab::GitAccessDesign,
- repository_resolver: -> (project) { ::DesignManagement::Repository.new(project) },
- suffix: :design
+ repository_resolver: -> (project) { project.design_management_repository.repository },
+ suffix: :design,
+ container_class: DesignManagement::Repository
).freeze
TYPES = {
diff --git a/lib/gitlab/gl_repository/repo_type.rb b/lib/gitlab/gl_repository/repo_type.rb
index 7792ef55b28..26b0ff86f67 100644
--- a/lib/gitlab/gl_repository/repo_type.rb
+++ b/lib/gitlab/gl_repository/repo_type.rb
@@ -55,11 +55,11 @@ module Gitlab
def repository_for(container)
return unless container
- repository_resolver.call(container)
+ repository_resolver.call(select_container(container))
end
def project_for(container)
- return container unless project_resolver
+ return select_container(container) unless project_resolver
project_resolver.call(container)
end
@@ -74,6 +74,10 @@ module Gitlab
private
+ def select_container(container)
+ container.is_a?(::DesignManagement::Repository) ? container.project : container
+ end
+
def default_container_class
Project
end
diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb
index c9766ee095a..904a2ccc79b 100644
--- a/lib/gitlab/gon_helper.rb
+++ b/lib/gitlab/gon_helper.rb
@@ -49,6 +49,7 @@ module Gitlab
gon.ee = Gitlab.ee?
gon.jh = Gitlab.jh?
gon.dot_com = Gitlab.com?
+ gon.uf_error_prefix = ::Gitlab::Utils::ErrorMessage::UF_ERROR_PREFIX
if current_user
gon.current_user_id = current_user.id
@@ -56,18 +57,19 @@ module Gitlab
gon.current_user_fullname = current_user.name
gon.current_user_avatar_url = current_user.avatar_url
gon.time_display_relative = current_user.time_display_relative
- gon.use_new_navigation = Feature.enabled?(:super_sidebar_nav, current_user) && current_user&.use_new_navigation
+ gon.use_new_navigation = NavHelper.show_super_sidebar?(current_user)
end
# Initialize gon.features with any flags that should be
# made globally available to the frontend
push_frontend_feature_flag(:usage_data_api, type: :ops)
push_frontend_feature_flag(:security_auto_fix)
- push_frontend_feature_flag(:new_header_search)
push_frontend_feature_flag(:source_editor_toolbar)
push_frontend_feature_flag(:vscode_web_ide, current_user)
- push_frontend_feature_flag(:integration_slack_app_notifications)
- push_frontend_feature_flag(:full_path_project_search, current_user)
+ push_frontend_feature_flag(:super_sidebar_peek, current_user)
+ push_frontend_feature_flag(:unbatch_graphql_queries, current_user)
+ # To be removed with https://gitlab.com/gitlab-org/gitlab/-/issues/399248
+ push_frontend_feature_flag(:remove_monitor_metrics)
end
# Exposes the state of a feature flag to the frontend code.
diff --git a/lib/gitlab/grape_logging/loggers/response_logger.rb b/lib/gitlab/grape_logging/loggers/response_logger.rb
index 0465f01f7f5..767c282d62e 100644
--- a/lib/gitlab/grape_logging/loggers/response_logger.rb
+++ b/lib/gitlab/grape_logging/loggers/response_logger.rb
@@ -8,7 +8,14 @@ module Gitlab
return {} unless Feature.enabled?(:log_response_length)
response_bytes = 0
- response.each { |resp| response_bytes += resp.to_s.bytesize }
+
+ case response
+ when String
+ response_bytes = response.bytesize
+ else
+ response.each { |resp| response_bytes += resp.to_s.bytesize }
+ end
+
{
response_bytes: response_bytes
}
diff --git a/lib/gitlab/graphql/authorize/authorize_resource.rb b/lib/gitlab/graphql/authorize/authorize_resource.rb
index 983bdb9c0a2..e3548b97ebf 100644
--- a/lib/gitlab/graphql/authorize/authorize_resource.rb
+++ b/lib/gitlab/graphql/authorize/authorize_resource.rb
@@ -45,8 +45,8 @@ module Gitlab
end
end
- def find_object(*args)
- raise NotImplementedError, "Implement #find_object in #{self.class.name}"
+ def find_object(id:)
+ GitlabSchema.find_by_gid(id)
end
def authorized_find!(*args, **kwargs)
diff --git a/lib/gitlab/graphql/deprecations/deprecation.rb b/lib/gitlab/graphql/deprecations/deprecation.rb
index 7f4cea7c635..dfcca5ee75b 100644
--- a/lib/gitlab/graphql/deprecations/deprecation.rb
+++ b/lib/gitlab/graphql/deprecations/deprecation.rb
@@ -9,7 +9,7 @@ module Gitlab
REASONS = {
REASON_RENAMED => 'This was renamed.',
- REASON_ALPHA => 'This feature is in Alpha. It can be changed or removed at any time.'
+ REASON_ALPHA => 'This feature is an Experiment. It can be changed or removed at any time.'
}.freeze
include ActiveModel::Validations
@@ -27,7 +27,7 @@ module Gitlab
return unless options
if alpha
- raise ArgumentError, '`alpha` and `deprecated` arguments cannot be passed at the same time' \
+ raise ArgumentError, '`experiment` and `deprecated` arguments cannot be passed at the same time' \
if deprecated
options[:reason] = :alpha
diff --git a/lib/gitlab/graphql/loaders/lazy_relation_loader.rb b/lib/gitlab/graphql/loaders/lazy_relation_loader.rb
new file mode 100644
index 00000000000..69056e87091
--- /dev/null
+++ b/lib/gitlab/graphql/loaders/lazy_relation_loader.rb
@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Graphql
+ module Loaders
+ class LazyRelationLoader
+ class << self
+ attr_accessor :model, :association
+
+ # Automatically register the inheriting
+ # classes to GitlabSchema as lazy objects.
+ def inherited(klass)
+ GitlabSchema.lazy_resolve(klass, :load)
+ end
+ end
+
+ def initialize(query_ctx, object, **kwargs)
+ @query_ctx = query_ctx
+ @object = object
+ @kwargs = kwargs
+
+ query_ctx[loader_cache_key] ||= Registry.new(relation(**kwargs))
+ query_ctx[loader_cache_key].register(object)
+ end
+
+ # Returns an instance of `RelationProxy` for the object (parent model).
+ # The returned object behaves like an Active Record relation to support
+ # keyset pagination.
+ def load
+ case reflection.macro
+ when :has_many
+ relation_proxy
+ when :has_one
+ relation_proxy.last
+ else
+ raise 'Not supported association type!'
+ end
+ end
+
+ private
+
+ attr_reader :query_ctx, :object, :kwargs
+
+ delegate :model, :association, to: :"self.class"
+
+ # Implement this one if you want to filter the relation
+ def relation(**)
+ base_relation
+ end
+
+ def loader_cache_key
+ @loader_cache_key ||= self.class.name.to_s + kwargs.sort.to_s
+ end
+
+ def base_relation
+ placeholder_record.association(association).scope
+ end
+
+ # This will only work for HasMany and HasOne associations for now
+ def placeholder_record
+ model.new(reflection.active_record_primary_key => 0)
+ end
+
+ def reflection
+ model.reflections[association.to_s]
+ end
+
+ def relation_proxy
+ RelationProxy.new(object, query_ctx[loader_cache_key])
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/graphql/loaders/lazy_relation_loader/registry.rb b/lib/gitlab/graphql/loaders/lazy_relation_loader/registry.rb
new file mode 100644
index 00000000000..ab2b2bd4dc2
--- /dev/null
+++ b/lib/gitlab/graphql/loaders/lazy_relation_loader/registry.rb
@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Graphql
+ module Loaders
+ class LazyRelationLoader
+ class Registry
+ PrematureQueryExecutionTriggered = Class.new(RuntimeError)
+ # Following methods are Active Record kicker methods which fire SQL query.
+ # We can support some of them with TopNLoader but for now restricting their use
+ # as we don't have a use case.
+ PROHIBITED_METHODS = (
+ ActiveRecord::FinderMethods.instance_methods(false) +
+ ActiveRecord::Calculations.instance_methods(false)
+ ).to_set.freeze
+
+ def initialize(relation)
+ @parents = []
+ @relation = relation
+ @records = []
+ @loaded = false
+ end
+
+ def register(object)
+ @parents << object
+ end
+
+ def method_missing(method_name, ...)
+ raise PrematureQueryExecutionTriggered if PROHIBITED_METHODS.include?(method_name)
+
+ result = relation.public_send(method_name, ...) # rubocop:disable GitlabSecurity/PublicSend
+
+ if result.is_a?(ActiveRecord::Relation) # Spawn methods generate a new relation (e.g. where, limit)
+ @relation = result
+
+ return self
+ end
+
+ result
+ end
+
+ def respond_to_missing?(method_name, include_private = false)
+ relation.respond_to?(method_name, include_private)
+ end
+
+ def load
+ return records if loaded
+
+ @loaded = true
+ @records = TopNLoader.load(relation, parents)
+ end
+
+ def for(object)
+ load.select { |record| record[foreign_key] == object[active_record_primary_key] }
+ .tap { |records| set_inverse_of(object, records) }
+ end
+
+ private
+
+ attr_reader :parents, :relation, :records, :loaded
+
+ delegate :proxy_association, to: :relation, private: true
+ delegate :reflection, to: :proxy_association, private: true
+ delegate :active_record_primary_key, :foreign_key, to: :reflection, private: true
+
+ def set_inverse_of(object, records)
+ records.each do |record|
+ object.association(reflection.name).set_inverse_instance(record)
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/graphql/loaders/lazy_relation_loader/relation_proxy.rb b/lib/gitlab/graphql/loaders/lazy_relation_loader/relation_proxy.rb
new file mode 100644
index 00000000000..bab2a272fb0
--- /dev/null
+++ b/lib/gitlab/graphql/loaders/lazy_relation_loader/relation_proxy.rb
@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Graphql
+ module Loaders
+ class LazyRelationLoader
+ # Proxies all the method calls to Registry instance.
+ # The main purpose of having this is that calling load
+ # on an instance of this class will only return the records
+ # associated with the main Active Record model.
+ class RelationProxy
+ def initialize(object, registry)
+ @object = object
+ @registry = registry
+ end
+
+ def load
+ registry.for(object)
+ end
+ alias_method :to_a, :load
+
+ def last(limit = 1)
+ result = registry.limit(limit)
+ .reverse_order!
+ .for(object)
+
+ return result.first if limit == 1 # This is the Active Record behavior
+
+ result
+ end
+
+ private
+
+ attr_reader :registry, :object
+
+ # Delegate everything to registry
+ def method_missing(method_name, ...)
+ result = registry.public_send(method_name, ...) # rubocop:disable GitlabSecurity/PublicSend
+
+ return self if result == registry
+
+ result
+ end
+
+ def respond_to_missing?(method_name, include_private = false)
+ registry.respond_to?(method_name, include_private)
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/graphql/loaders/lazy_relation_loader/top_n_loader.rb b/lib/gitlab/graphql/loaders/lazy_relation_loader/top_n_loader.rb
new file mode 100644
index 00000000000..6404148832b
--- /dev/null
+++ b/lib/gitlab/graphql/loaders/lazy_relation_loader/top_n_loader.rb
@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+# rubocop:disable CodeReuse/ActiveRecord
+module Gitlab
+ module Graphql
+ module Loaders
+ class LazyRelationLoader
+ # Loads the top-n records for each given parent record.
+ # For example; if you want to load only 5 confidential issues ordered by
+ # their updated_at column per project for a list of projects by issuing only a single
+ # SQL query then this class can help you.
+ # Note that the limit applies per parent record which means that if you apply limit as 5
+ # for 10 projects, this loader will load 50 records in total.
+ class TopNLoader
+ def self.load(original_relation, parents)
+ new(original_relation, parents).load
+ end
+
+ def initialize(original_relation, parents)
+ @original_relation = original_relation
+ @parents = parents
+ end
+
+ def load
+ klass.select(klass.arel_table[Arel.star])
+ .from(from)
+ .joins("JOIN LATERAL (#{lateral_relation.to_sql}) AS #{klass.arel_table.name} ON true")
+ .includes(original_includes)
+ .preload(original_preload)
+ .eager_load(original_eager_load)
+ .load
+ end
+
+ private
+
+ attr_reader :original_relation, :parents
+
+ delegate :proxy_association, to: :original_relation, private: true
+ delegate :reflection, to: :proxy_association, private: true
+ delegate :klass, :foreign_key, :active_record, :active_record_primary_key,
+ to: :reflection, private: true
+
+ # This only works for HasMany and HasOne.
+ def lateral_relation
+ original_relation
+ .unscope(where: foreign_key) # unscoping the where condition generated for the placeholder_record.
+ .where(klass.arel_table[foreign_key].eq(active_record.arel_table[active_record_primary_key]))
+ end
+
+ def from
+ grouping_arel_node.as("#{active_record.arel_table.name}(#{active_record.primary_key})")
+ end
+
+ def grouping_arel_node
+ Arel::Nodes::Grouping.new(id_list_arel_node)
+ end
+
+ def id_list_arel_node
+ parent_ids.map { |id| [id] }
+ .then { |ids| Arel::Nodes::ValuesList.new(ids) }
+ end
+
+ def parent_ids
+ parents.pluck(active_record.primary_key)
+ end
+
+ def original_includes
+ original_relation.includes_values
+ end
+
+ def original_preload
+ original_relation.preload_values
+ end
+
+ def original_eager_load
+ original_relation.eager_load_values
+ end
+ end
+ end
+ end
+ end
+end
+# rubocop:enable CodeReuse/ActiveRecord
diff --git a/lib/gitlab/graphql/pagination/connections.rb b/lib/gitlab/graphql/pagination/connections.rb
index 965c01dd02f..df1231b005f 100644
--- a/lib/gitlab/graphql/pagination/connections.rb
+++ b/lib/gitlab/graphql/pagination/connections.rb
@@ -14,6 +14,10 @@ module Gitlab
Gitlab::Graphql::Pagination::Keyset::Connection)
schema.connections.add(
+ Gitlab::Graphql::Loaders::LazyRelationLoader::RelationProxy,
+ Gitlab::Graphql::Pagination::Keyset::Connection)
+
+ schema.connections.add(
Gitlab::Graphql::ExternallyPaginatedArray,
Gitlab::Graphql::Pagination::ExternallyPaginatedArrayConnection)
diff --git a/lib/gitlab/graphql/project/dast_profile_connection_extension.rb b/lib/gitlab/graphql/project/dast_profile_connection_extension.rb
index 45f90de2f17..1c21d286187 100644
--- a/lib/gitlab/graphql/project/dast_profile_connection_extension.rb
+++ b/lib/gitlab/graphql/project/dast_profile_connection_extension.rb
@@ -12,9 +12,12 @@ module Gitlab
def preload_authorizations(dast_profiles)
return unless dast_profiles
- projects = dast_profiles.map(&:project)
- users = dast_profiles.filter_map { |dast_profile| dast_profile.dast_profile_schedule&.owner }
- Preloaders::UsersMaxAccessLevelInProjectsPreloader.new(projects: projects, users: users).execute
+ project_users = dast_profiles.group_by(&:project).transform_values do |project_profiles|
+ project_profiles
+ .filter_map { |profile| profile.dast_profile_schedule&.owner }
+ .uniq
+ end
+ Preloaders::UsersMaxAccessLevelByProjectPreloader.new(project_users: project_users).execute
end
end
end
diff --git a/lib/gitlab/graphql/subscriptions/action_cable_with_load_balancing.rb b/lib/gitlab/graphql/subscriptions/action_cable_with_load_balancing.rb
new file mode 100644
index 00000000000..851750163af
--- /dev/null
+++ b/lib/gitlab/graphql/subscriptions/action_cable_with_load_balancing.rb
@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Graphql
+ module Subscriptions
+ class ActionCableWithLoadBalancing < ::GraphQL::Subscriptions::ActionCableSubscriptions
+ extend ::Gitlab::Utils::Override
+ include Gitlab::Database::LoadBalancing::WalTrackingSender
+ include Gitlab::Database::LoadBalancing::WalTrackingReceiver
+
+ KEY_PAYLOAD = 'gql_payload'
+ KEY_WAL_LOCATIONS = 'wal_locations'
+
+ override :execute_all
+ def execute_all(event, object)
+ super(event, {
+ KEY_WAL_LOCATIONS => current_wal_locations,
+ KEY_PAYLOAD => object
+ })
+ end
+
+ # We fall back to the primary in case no replica is sufficiently caught up.
+ override :execute_update
+ def execute_update(subscription_id, event, object)
+ # Make sure we do not accidentally try to unwrap messages that are not wrapped.
+ # This could in theory happen if workers roll over where some send wrapped payload
+ # and others expect the original payload.
+ return super(subscription_id, event, object) unless wrapped_payload?(object)
+
+ wal_locations = object[KEY_WAL_LOCATIONS]
+ ::Gitlab::Database::LoadBalancing::Session.current.use_primary! if use_primary?(wal_locations)
+
+ super(subscription_id, event, object[KEY_PAYLOAD])
+ end
+
+ private
+
+ def wrapped_payload?(object)
+ object.try(:key?, KEY_PAYLOAD)
+ end
+
+ def use_primary?(wal_locations)
+ wal_locations.blank? || !databases_in_sync?(wal_locations)
+ end
+
+ # We stringify keys since otherwise the graphql-ruby serializer will inject additional metadata
+ # to keep track of which keys used to be symbols.
+ def current_wal_locations
+ wal_locations_by_db_name&.stringify_keys
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/harbor/client.rb b/lib/gitlab/harbor/client.rb
index ee40725ba95..380e4e42bc7 100644
--- a/lib/gitlab/harbor/client.rb
+++ b/lib/gitlab/harbor/client.rb
@@ -14,9 +14,9 @@ module Gitlab
@integration = integration
end
- def ping
- options = { headers: headers.merge!('Accept': 'text/plain') }
- response = Gitlab::HTTP.get(url('ping'), options)
+ def check_project_availability
+ options = { headers: headers.merge!('Accept': 'application/json') }
+ response = Gitlab::HTTP.head(url("projects?project_name=#{integration.project_name}"), options)
{ success: response.success? }
end
diff --git a/lib/gitlab/hook_data/base_builder.rb b/lib/gitlab/hook_data/base_builder.rb
index e5bae61ae4e..4a81f6b8a0e 100644
--- a/lib/gitlab/hook_data/base_builder.rb
+++ b/lib/gitlab/hook_data/base_builder.rb
@@ -5,15 +5,14 @@ module Gitlab
class BaseBuilder
attr_accessor :object
- MARKDOWN_SIMPLE_IMAGE = %r{
- #{::Gitlab::Regex.markdown_code_or_html_blocks}
- |
- (?<image>
- !
- \[(?<title>[^\n]*?)\]
- \((?<url>(?!(https?://|//))[^\n]+?)\)
- )
- }mx.freeze
+ MARKDOWN_SIMPLE_IMAGE =
+ "#{::Gitlab::Regex.markdown_code_or_html_blocks_untrusted}" \
+ '|' \
+ '(?P<image>' \
+ '!' \
+ '\[(?P<title>[^\n]*?)\]' \
+ '\((?P<url>(?P<https>(https?://|//)?)[^\n]+?)\)' \
+ ')'.freeze
def initialize(object)
@object = object
@@ -37,15 +36,18 @@ module Gitlab
def absolute_image_urls(markdown_text)
return markdown_text unless markdown_text.present?
- markdown_text.gsub(MARKDOWN_SIMPLE_IMAGE) do
- if $~[:image]
- url = $~[:url]
+ regex = Gitlab::UntrustedRegexp.new(MARKDOWN_SIMPLE_IMAGE, multiline: false)
+ return markdown_text unless regex.match?(markdown_text)
+
+ regex.replace_gsub(markdown_text) do |match|
+ if match[:image] && !match[:https]
+ url = match[:url]
url = "#{uploads_prefix}#{url}" if url.start_with?('/uploads')
url = "/#{url}" unless url.start_with?('/')
- "![#{$~[:title]}](#{Gitlab.config.gitlab.url}#{url})"
+ "![#{match[:title]}](#{Gitlab.config.gitlab.url}#{url})"
else
- $~[0]
+ match.to_s
end
end
end
diff --git a/lib/gitlab/http_connection_adapter.rb b/lib/gitlab/http_connection_adapter.rb
index c6f9f2df299..afb740a902b 100644
--- a/lib/gitlab/http_connection_adapter.rb
+++ b/lib/gitlab/http_connection_adapter.rb
@@ -24,11 +24,18 @@ module Gitlab
override :connection
def connection
- @uri, hostname = validate_url!(uri)
+ result = validate_url_with_proxy!(uri)
+ @uri = result.uri
+ hostname = result.hostname
http = super
http.hostname_override = hostname if hostname
+ unless result.use_proxy
+ http.proxy_from_env = false
+ http.proxy_address = nil
+ end
+
gitlab_http = Gitlab::NetHttpAdapter.new(http.address, http.port)
http.instance_variables.each do |variable|
@@ -40,12 +47,13 @@ module Gitlab
private
- def validate_url!(url)
- Gitlab::UrlBlocker.validate!(url, allow_local_network: allow_local_requests?,
- allow_localhost: allow_local_requests?,
- allow_object_storage: allow_object_storage?,
- dns_rebind_protection: dns_rebind_protection?,
- schemes: %w[http https])
+ def validate_url_with_proxy!(url)
+ Gitlab::UrlBlocker.validate_url_with_proxy!(
+ url, allow_local_network: allow_local_requests?,
+ allow_localhost: allow_local_requests?,
+ allow_object_storage: allow_object_storage?,
+ dns_rebind_protection: dns_rebind_protection?,
+ schemes: %w[http https])
rescue Gitlab::UrlBlocker::BlockedUrlError => e
raise Gitlab::HTTP::BlockedUrlError, "URL is blocked: #{e.message}"
end
@@ -59,8 +67,6 @@ module Gitlab
end
def dns_rebind_protection?
- return false if Gitlab.http_proxy_env?
-
Gitlab::CurrentSettings.dns_rebinding_protection_enabled?
end
diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb
index 8fe5868ca57..fa85d839927 100644
--- a/lib/gitlab/i18n.rb
+++ b/lib/gitlab/i18n.rb
@@ -44,30 +44,30 @@ module Gitlab
TRANSLATION_LEVELS = {
'bg' => 0,
'cs_CZ' => 0,
- 'da_DK' => 34,
- 'de' => 16,
+ 'da_DK' => 31,
+ 'de' => 15,
'en' => 100,
'eo' => 0,
- 'es' => 33,
+ 'es' => 31,
'fil_PH' => 0,
- 'fr' => 99,
+ 'fr' => 98,
'gl_ES' => 0,
'id_ID' => 0,
'it' => 1,
- 'ja' => 31,
- 'ko' => 20,
+ 'ja' => 77,
+ 'ko' => 18,
'nb_NO' => 23,
'nl_NL' => 0,
'pl_PL' => 3,
- 'pt_BR' => 57,
- 'ro_RO' => 91,
- 'ru' => 26,
- 'si_LK' => 11,
- 'tr_TR' => 10,
- 'uk' => 55,
- 'zh_CN' => 98,
+ 'pt_BR' => 56,
+ 'ro_RO' => 84,
+ 'ru' => 24,
+ 'si_LK' => 10,
+ 'tr_TR' => 9,
+ 'uk' => 54,
+ 'zh_CN' => 99,
'zh_HK' => 1,
- 'zh_TW' => 98
+ 'zh_TW' => 99
}.freeze
private_constant :TRANSLATION_LEVELS
@@ -118,12 +118,17 @@ module Gitlab
end
def setup(domain:, default_locale:)
+ custom_pluralization
setup_repositories(domain)
setup_default_locale(default_locale)
end
private
+ def custom_pluralization
+ Gitlab::I18n::Pluralization.install_on(FastGettext)
+ end
+
def setup_repositories(domain)
translation_repositories = [
(po_repository(domain, 'jh/locale') if Gitlab.jh?),
diff --git a/lib/gitlab/i18n/pluralization.rb b/lib/gitlab/i18n/pluralization.rb
new file mode 100644
index 00000000000..5d4a05f1fd1
--- /dev/null
+++ b/lib/gitlab/i18n/pluralization.rb
@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module I18n
+ # Pluralization formulas per locale used by FastGettext via:
+ # `FastGettext.pluralisation_rule.call(count)`.
+ module Pluralization
+ # rubocop:disable all
+ MAP = {
+ "bg" => ->(n) { (n != 1) },
+ "cs_CZ" => ->(n) { (n==1) ? 0 : (n>=2 && n<=4) ? 1 : 3 },
+ "da_DK" => ->(n) { (n != 1) },
+ "de" => ->(n) { (n != 1) },
+ "en" => ->(n) { (n != 1) },
+ "eo" => ->(n) { (n != 1) },
+ "es" => ->(n) { (n != 1) },
+ "fil_PH" => ->(n) { (n > 1) },
+ "fr" => ->(n) { (n > 1) },
+ "gl_ES" => ->(n) { (n != 1) },
+ "id_ID" => ->(n) { 0 },
+ "it" => ->(n) { (n != 1) },
+ "ja" => ->(n) { 0 },
+ "ko" => ->(n) { 0 },
+ "nb_NO" => ->(n) { (n != 1) },
+ "nl_NL" => ->(n) { (n != 1) },
+ "pl_PL" => ->(n) { (n==1 ? 0 : (n%10>=2 && n%10<=4) && (n%100<12 || n%100>14) ? 1 : n!=1 && (n%10>=0 && n%10<=1) || (n%10>=5 && n%10<=9) || (n%100>=12 && n%100<=14) ? 2 : 3) },
+ "pt_BR" => ->(n) { (n != 1) },
+ "ro_RO" => ->(n) { (n==1 ? 0 : (n==0 || (n%100>0 && n%100<20)) ? 1 : 2) },
+ "ru" => ->(n) { ((n%10==1 && n%100!=11) ? 0 : ((n%10 >= 2 && n%10 <=4 && (n%100 < 12 || n%100 > 14)) ? 1 : ((n%10 == 0 || (n%10 >= 5 && n%10 <=9)) || (n%100 >= 11 && n%100 <= 14)) ? 2 : 3)) },
+ "si_LK" => ->(n) { (n != 1) },
+ "tr_TR" => ->(n) { (n != 1) },
+ "uk" => ->(n) { ((n%10==1 && n%100!=11) ? 0 : ((n%10 >= 2 && n%10 <=4 && (n%100 < 12 || n%100 > 14)) ? 1 : ((n%10 == 0 || (n%10 >= 5 && n%10 <=9)) || (n%100 >= 11 && n%100 <= 14)) ? 2 : 3)) },
+ "zh_CN" => ->(n) { 0 },
+ "zh_HK" => ->(n) { 0 },
+ "zh_TW" => ->(n) { 0 }
+ }.freeze
+ # rubocop:enable
+
+ NOT_FOUND_ERROR = lambda do |locale|
+ po = File.expand_path("../../../locale/#{locale}/gitlab.po", __dir__)
+
+ forms = File.read(po)[/Plural-Forms:.*; plural=(.*?);\\n/, 1] if File.exist?(po)
+ suggestion = <<~TEXT if forms
+ Add the following line to #{__FILE__}:
+
+ MAP = {
+ ...
+ "#{locale}" => ->(n) { #{forms} },
+ ...
+ }.freeze
+
+ This rule was extracted from #{po}.
+ TEXT
+
+ raise ArgumentError, <<~MESSAGE
+ Missing pluralization rule for locale #{locale.inspect}.
+
+ #{suggestion}
+ MESSAGE
+ end
+
+ def self.call(count)
+ locale = FastGettext.locale
+
+ MAP.fetch(locale, &NOT_FOUND_ERROR).call(count)
+ end
+
+ def self.install_on(klass)
+ klass.extend(FastGettextClassMethods)
+ end
+
+ module FastGettextClassMethods
+ # FastGettext allows to set the rule via
+ # `FastGettext.pluralisation_rule=` which is on thread-level.
+ #
+ # Because we are patching FastGettext at boot time per thread values
+ # won't work so we have to override the method implementation.
+ #
+ # `FastGettext.pluralisation_rule=` has now no effect.
+ def pluralisation_rule
+ Gitlab::I18n::Pluralization
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/import/errors.rb b/lib/gitlab/import/errors.rb
new file mode 100644
index 00000000000..b9e8c9135fd
--- /dev/null
+++ b/lib/gitlab/import/errors.rb
@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Import
+ module Errors
+ # Merges all nested subrelation errors into base errors object.
+ #
+ # @example
+ # issue = Project.last.issues.new(
+ # title: 'test',
+ # author: User.first,
+ # notes: [Note.new(
+ # award_emoji: [AwardEmoji.new(name: 'test')]
+ # )])
+ #
+ # issue.validate
+ # issue.errors.full_messages
+ # => ["Notes is invalid"]
+ #
+ # Gitlab::Import::Errors.merge_nested_errors(issue)
+ # issue.errors.full_messages
+ # => ["Notes is invalid",
+ # "Award emoji is invalid",
+ # "Awardable can't be blank",
+ # "Name is not a valid emoji name",
+ # ...
+ # ]
+ def self.merge_nested_errors(object)
+ object.errors.each do |error|
+ association = object.class.reflect_on_association(error.attribute)
+
+ next unless association&.collection?
+
+ records = object.public_send(error.attribute).select(&:invalid?) # rubocop: disable GitlabSecurity/PublicSend
+
+ records.each do |record|
+ merge_nested_errors(record)
+
+ object.errors.merge!(record.errors)
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/import/import_failure_service.rb b/lib/gitlab/import/import_failure_service.rb
index bebd64b29a9..714d9b3edd9 100644
--- a/lib/gitlab/import/import_failure_service.rb
+++ b/lib/gitlab/import/import_failure_service.rb
@@ -50,10 +50,10 @@ module Gitlab
def execute
track_exception
- persist_failure
-
- track_metrics if metrics
- import_state.mark_as_failed(exception.message) if fail_import
+ persist_failure.tap do
+ import_state.mark_as_failed(exception.message) if fail_import
+ track_metrics if metrics
+ end
end
private
diff --git a/lib/gitlab/import/metrics.rb b/lib/gitlab/import/metrics.rb
index 7a0cf1682a6..8263df3dc37 100644
--- a/lib/gitlab/import/metrics.rb
+++ b/lib/gitlab/import/metrics.rb
@@ -32,6 +32,14 @@ module Gitlab
return unless project.github_import?
track_usage_event(:github_import_project_failure, project.id)
+ track_import_state('github', 'Import::GithubService')
+ end
+
+ def track_canceled_import
+ return unless project.github_import?
+
+ track_usage_event(:github_import_project_cancelled, project.id)
+ track_import_state('github', 'Import::GithubService')
end
def issues_counter
@@ -75,7 +83,25 @@ module Gitlab
def track_finish_metric
return unless project.github_import?
- track_usage_event(:github_import_project_success, project.id)
+ track_import_state('github', 'Import::GithubService')
+
+ case project.beautified_import_status_name
+ when 'partially completed'
+ track_usage_event(:github_import_project_partially_completed, project.id)
+ when 'completed'
+ track_usage_event(:github_import_project_success, project.id)
+ end
+ end
+
+ def track_import_state(type, category)
+ Gitlab::Tracking.event(
+ category,
+ 'create',
+ label: "#{type}_import_project_state",
+ project: project,
+ import_type: type,
+ state: project.beautified_import_status_name
+ )
end
end
end
diff --git a/lib/gitlab/import_export/attributes_finder.rb b/lib/gitlab/import_export/attributes_finder.rb
index 8843b4f5755..dea989931c7 100644
--- a/lib/gitlab/import_export/attributes_finder.rb
+++ b/lib/gitlab/import_export/attributes_finder.rb
@@ -3,7 +3,8 @@
module Gitlab
module ImportExport
class AttributesFinder
- attr_reader :tree, :included_attributes, :excluded_attributes, :methods, :preloads, :export_reorders
+ attr_reader :tree, :included_attributes, :excluded_attributes, :methods, :preloads, :export_reorders,
+ :import_only_tree
def initialize(config:)
@tree = config[:tree] || {}
@@ -13,13 +14,16 @@ module Gitlab
@preloads = config[:preloads] || {}
@export_reorders = config[:export_reorders] || {}
@include_if_exportable = config[:include_if_exportable] || {}
+ @import_only_tree = config[:import_only_tree] || {}
end
def find_root(model_key)
find(model_key, @tree[model_key])
end
- def find_relations_tree(model_key)
+ def find_relations_tree(model_key, include_import_only_tree: false)
+ return @tree[model_key].deep_merge(@import_only_tree[model_key] || {}) if include_import_only_tree
+
@tree[model_key]
end
diff --git a/lib/gitlab/import_export/attributes_permitter.rb b/lib/gitlab/import_export/attributes_permitter.rb
index 8c7a6c13246..889cab88de4 100644
--- a/lib/gitlab/import_export/attributes_permitter.rb
+++ b/lib/gitlab/import_export/attributes_permitter.rb
@@ -80,7 +80,7 @@ module Gitlab
# Deep traverse relations tree to build a list of allowed model relations
def build_associations
- stack = @attributes_finder.tree.to_a
+ stack = @attributes_finder.tree.deep_merge(@attributes_finder.import_only_tree).to_a
while stack.any?
model_name, relations = stack.pop
diff --git a/lib/gitlab/import_export/base/relation_factory.rb b/lib/gitlab/import_export/base/relation_factory.rb
index e3813070aa4..3d96e891797 100644
--- a/lib/gitlab/import_export/base/relation_factory.rb
+++ b/lib/gitlab/import_export/base/relation_factory.rb
@@ -295,6 +295,13 @@ module Gitlab
end
def unique_relation?
+ # this guard is necessary because
+ # when multiple approval_project_rules_protected_branch referenced the same protected branch
+ # or approval_project_rules_user referenced the same user
+ # the different instances were squashed into one
+ # because this method returned true for reason that needs investigation
+ return if @relation_sym == :approval_rules
+
strong_memoize(:unique_relation) do
importable_foreign_key.present? &&
(has_unique_index_on_importable_fk? || uses_importable_fk_as_primary_key?)
diff --git a/lib/gitlab/import_export/base/relation_object_saver.rb b/lib/gitlab/import_export/base/relation_object_saver.rb
index 77b85fc9f15..986191bdb6b 100644
--- a/lib/gitlab/import_export/base/relation_object_saver.rb
+++ b/lib/gitlab/import_export/base/relation_object_saver.rb
@@ -17,6 +17,8 @@ module Gitlab
BATCH_SIZE = 100
MIN_RECORDS_SIZE = 1
+ attr_reader :invalid_subrelations
+
# @param relation_object [Object] Object of a project/group, e.g. an issue
# @param relation_key [String] Name of the object association to group/project, e.g. :issues
# @param relation_definition [Hash] Object subrelations as defined in import_export.yml
@@ -43,14 +45,11 @@ module Gitlab
relation_object.save!
save_subrelations
- ensure
- log_invalid_subrelations
end
private
- attr_reader :relation_object, :relation_key, :relation_definition,
- :importable, :collection_subrelations, :invalid_subrelations
+ attr_reader :relation_object, :relation_key, :relation_definition, :importable, :collection_subrelations
# rubocop:disable GitlabSecurity/PublicSend
def save_subrelations
@@ -92,30 +91,6 @@ module Gitlab
end
end
# rubocop:enable GitlabSecurity/PublicSend
-
- def log_invalid_subrelations
- invalid_subrelations.flatten.each do |record|
- Gitlab::Import::Logger.info(
- message: '[Project/Group Import] Invalid subrelation',
- importable_column_name => importable.id,
- relation_key: relation_key,
- error_messages: record.errors.full_messages.to_sentence
- )
-
- ImportFailure.create(
- source: 'RelationObjectSaver#save!',
- relation_key: relation_key,
- exception_class: 'RecordInvalid',
- exception_message: record.errors.full_messages.to_sentence,
- correlation_id_value: Labkit::Correlation::CorrelationId.current_or_new_id,
- importable_column_name => importable.id
- )
- end
- end
-
- def importable_column_name
- @column_name ||= importable.class.reflect_on_association(:import_failures).foreign_key.to_sym
- end
end
end
end
diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb
index 64ef3dd4830..d681f39f00b 100644
--- a/lib/gitlab/import_export/command_line_util.rb
+++ b/lib/gitlab/import_export/command_line_util.rb
@@ -90,6 +90,7 @@ module Gitlab
def untar_with_options(archive:, dir:, options:)
execute_cmd(%W(tar -#{options} #{archive} -C #{dir}))
execute_cmd(%W(chmod -R #{UNTAR_MASK} #{dir}))
+ remove_symlinks(dir)
end
# rubocop:disable Gitlab/ModuleWithInstanceVariables
@@ -120,6 +121,19 @@ module Gitlab
FileUtils.copy_entry(source, destination)
true
end
+
+ def remove_symlinks(dir)
+ ignore_file_names = %w[. ..]
+
+ # Using File::FNM_DOTMATCH to also delete symlinks starting with "."
+ Dir.glob("#{dir}/**/*", File::FNM_DOTMATCH)
+ .reject { |f| ignore_file_names.include?(File.basename(f)) }
+ .each do |filepath|
+ FileUtils.rm(filepath) if File.lstat(filepath).symlink?
+ end
+
+ true
+ end
end
end
end
diff --git a/lib/gitlab/import_export/config.rb b/lib/gitlab/import_export/config.rb
index 83c4bc47349..e1a62e3b25a 100644
--- a/lib/gitlab/import_export/config.rb
+++ b/lib/gitlab/import_export/config.rb
@@ -10,6 +10,7 @@ module Gitlab
@ee_hash = @hash.delete(:ee) || {}
@hash[:tree] = normalize_tree(@hash[:tree])
+ @hash[:import_only_tree] = normalize_tree(@hash[:import_only_tree] || {})
@ee_hash[:tree] = normalize_tree(@ee_hash[:tree] || {})
end
@@ -51,7 +52,7 @@ module Gitlab
end
def parse_yaml
- YAML.load_file(@config)
+ YAML.safe_load_file(@config, aliases: true, permitted_classes: [Symbol])
end
end
end
diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb
index 1878b5b1a30..d2593289c23 100644
--- a/lib/gitlab/import_export/file_importer.rb
+++ b/lib/gitlab/import_export/file_importer.rb
@@ -8,7 +8,6 @@ module Gitlab
ImporterError = Class.new(StandardError)
MAX_RETRIES = 8
- IGNORED_FILENAMES = %w(. ..).freeze
def self.import(*args, **kwargs)
new(*args, **kwargs).import
@@ -24,7 +23,7 @@ module Gitlab
mkdir_p(@shared.export_path)
mkdir_p(@shared.archive_path)
- remove_symlinks
+ remove_symlinks(@shared.export_path)
copy_archive
wait_for_archived_file do
@@ -36,7 +35,7 @@ module Gitlab
false
ensure
remove_import_file
- remove_symlinks
+ remove_symlinks(@shared.export_path)
end
private
@@ -86,22 +85,10 @@ module Gitlab
end
end
- def remove_symlinks
- extracted_files.each do |path|
- FileUtils.rm(path) if File.lstat(path).symlink?
- end
-
- true
- end
-
def remove_import_file
FileUtils.rm_rf(@archive_file)
end
- def extracted_files
- Dir.glob("#{@shared.export_path}/**/*", File::FNM_DOTMATCH).reject { |f| IGNORED_FILENAMES.include?(File.basename(f)) }
- end
-
def validate_decompressed_archive_size
raise ImporterError, _('Decompressed archive size validation failed.') unless size_validator.valid?
end
diff --git a/lib/gitlab/import_export/group/relation_tree_restorer.rb b/lib/gitlab/import_export/group/relation_tree_restorer.rb
index 5a78f2fb531..5453792e904 100644
--- a/lib/gitlab/import_export/group/relation_tree_restorer.rb
+++ b/lib/gitlab/import_export/group/relation_tree_restorer.rb
@@ -34,7 +34,6 @@ module Gitlab
update_params!
BulkInsertableAssociations.with_bulk_insert(enabled: bulk_insert_enabled) do
- fix_ci_pipelines_not_sorted_on_legacy_project_json!
create_relations!
end
end
@@ -90,13 +89,23 @@ module Gitlab
def save_relation_object(relation_object, relation_key, relation_definition, relation_index)
if relation_object.new_record?
- Gitlab::ImportExport::Base::RelationObjectSaver.new(
+ saver = Gitlab::ImportExport::Base::RelationObjectSaver.new(
relation_object: relation_object,
relation_key: relation_key,
relation_definition: relation_definition,
importable: @importable
- ).execute
+ )
+
+ saver.execute
+
+ log_invalid_subrelations(saver.invalid_subrelations, relation_key)
else
+ if relation_object.invalid?
+ Gitlab::Import::Errors.merge_nested_errors(relation_object)
+
+ raise(ActiveRecord::RecordInvalid, relation_object)
+ end
+
import_failure_service.with_retry(action: 'relation_object.save!', relation_key: relation_key, relation_index: relation_index) do
relation_object.save!
end
@@ -113,7 +122,7 @@ module Gitlab
@relations ||=
@reader
.attributes_finder
- .find_relations_tree(importable_class_sym)
+ .find_relations_tree(importable_class_sym, include_import_only_tree: true)
.deep_stringify_keys
end
@@ -126,9 +135,7 @@ module Gitlab
modify_attributes
- Gitlab::Timeless.timeless(@importable) do
- @importable.save!
- end
+ @importable.save!(touch: false)
end
def filter_attributes(params)
@@ -265,15 +272,6 @@ module Gitlab
}
end
- # Temporary fix for https://gitlab.com/gitlab-org/gitlab/-/issues/27883 when import from legacy project.json
- # This should be removed once legacy JSON format is deprecated.
- # Ndjson export file will fix the order during project export.
- def fix_ci_pipelines_not_sorted_on_legacy_project_json!
- return unless @relation_reader.legacy?
-
- @relation_reader.sort_ci_pipelines_by_id
- end
-
# Enable logging of each top-level relation creation when Importing into a Group
def log_relation_creation(importable, relation_key, relation_object)
root_ancestor_group = importable.try(:root_ancestor)
@@ -290,6 +288,32 @@ module Gitlab
message: '[Project/Group Import] Created new object relation'
)
end
+
+ def log_invalid_subrelations(invalid_subrelations, relation_key)
+ invalid_subrelations.flatten.each do |record|
+ Gitlab::Import::Errors.merge_nested_errors(record)
+
+ @shared.logger.info(
+ message: '[Project/Group Import] Invalid subrelation',
+ importable_column_name => @importable.id,
+ relation_key: relation_key,
+ error_messages: record.errors.full_messages.to_sentence
+ )
+
+ ::ImportFailure.create(
+ source: 'RelationTreeRestorer#save_relation_object',
+ relation_key: relation_key,
+ exception_class: 'ActiveRecord::RecordInvalid',
+ exception_message: record.errors.full_messages.to_sentence,
+ correlation_id_value: Labkit::Correlation::CorrelationId.current_or_new_id,
+ importable_column_name => @importable.id
+ )
+ end
+ end
+
+ def importable_column_name
+ @column_name ||= @importable.class.reflect_on_association(:import_failures).foreign_key.to_sym
+ end
end
end
end
diff --git a/lib/gitlab/import_export/json/legacy_reader.rb b/lib/gitlab/import_export/json/legacy_reader.rb
deleted file mode 100644
index ee360020556..00000000000
--- a/lib/gitlab/import_export/json/legacy_reader.rb
+++ /dev/null
@@ -1,123 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module ImportExport
- module Json
- class LegacyReader
- class File < LegacyReader
- include Gitlab::Utils::StrongMemoize
-
- def initialize(path, relation_names:, allowed_path: nil)
- @path = path
- super(
- relation_names: relation_names,
- allowed_path: allowed_path)
- end
-
- def exist?
- ::File.exist?(@path)
- end
-
- protected
-
- def tree_hash
- strong_memoize(:tree_hash) do
- read_hash
- end
- end
-
- def read_hash
- Gitlab::Json.parse(::File.read(@path))
- rescue StandardError => e
- Gitlab::ErrorTracking.log_exception(e)
- raise Gitlab::ImportExport::Error, 'Incorrect JSON format'
- end
- end
-
- class Hash < LegacyReader
- def initialize(tree_hash, relation_names:, allowed_path: nil)
- @tree_hash = tree_hash
- super(
- relation_names: relation_names,
- allowed_path: allowed_path)
- end
-
- def exist?
- @tree_hash.present?
- end
-
- protected
-
- attr_reader :tree_hash
- end
-
- def initialize(relation_names:, allowed_path:)
- @relation_names = relation_names.map(&:to_s)
- @consumed_relations = Set.new
-
- # This is legacy reader, to be used in transition
- # period before `.ndjson`,
- # we strong validate what is being readed
- @allowed_path = allowed_path
- end
-
- def exist?
- raise NotImplementedError
- end
-
- def legacy?
- true
- end
-
- def consume_attributes(importable_path)
- unless importable_path == @allowed_path
- raise ArgumentError, "Invalid #{importable_path} passed to `consume_attributes`. Use #{@allowed_path} instead."
- end
-
- attributes
- end
-
- def consume_relation(importable_path, key)
- unless importable_path == @allowed_path
- raise ArgumentError, "Invalid #{importable_name} passed to `consume_relation`. Use #{@allowed_path} instead."
- end
-
- Enumerator.new do |documents|
- next unless @consumed_relations.add?("#{importable_path}/#{key}")
-
- value = relations.delete(key)
- next if value.nil?
-
- if value.is_a?(Array)
- value.each.with_index do |item, idx|
- documents << [item, idx]
- end
- else
- documents << [value, 0]
- end
- end
- end
-
- def sort_ci_pipelines_by_id
- relations['ci_pipelines']&.sort_by! { |hash| hash['id'] }
- end
-
- private
-
- attr_reader :relation_names, :allowed_path
-
- def tree_hash
- raise NotImplementedError
- end
-
- def attributes
- @attributes ||= tree_hash.slice!(*relation_names)
- end
-
- def relations
- @relations ||= tree_hash.extract!(*relation_names)
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/import_export/json/legacy_writer.rb b/lib/gitlab/import_export/json/legacy_writer.rb
deleted file mode 100644
index e03ab9f7650..00000000000
--- a/lib/gitlab/import_export/json/legacy_writer.rb
+++ /dev/null
@@ -1,88 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module ImportExport
- module Json
- class LegacyWriter
- include Gitlab::ImportExport::CommandLineUtil
-
- attr_reader :path
-
- def initialize(path, allowed_path:)
- @path = path
- @keys = Set.new
-
- # This is legacy writer, to be used in transition
- # period before `.ndjson`,
- # we strong validate what is being written
- @allowed_path = allowed_path
-
- mkdir_p(File.dirname(@path))
- file.write('{}')
- end
-
- def close
- @file&.close
- @file = nil
- end
-
- def write_attributes(exportable_path, hash)
- unless exportable_path == @allowed_path
- raise ArgumentError, "Invalid #{exportable_path}"
- end
-
- hash.each do |key, value|
- write(key, value)
- end
- end
-
- def write_relation(exportable_path, key, value)
- unless exportable_path == @allowed_path
- raise ArgumentError, "Invalid #{exportable_path}"
- end
-
- write(key, value)
- end
-
- def write_relation_array(exportable_path, key, items)
- unless exportable_path == @allowed_path
- raise ArgumentError, "Invalid #{exportable_path}"
- end
-
- write(key, [])
-
- # rewind by two bytes, to overwrite ']}'
- file.pos = file.size - 2
-
- items.each_with_index do |item, idx|
- file.write(',') if idx > 0
- file.write(item.to_json)
- end
-
- file.write(']}')
- end
-
- private
-
- def write(key, value)
- raise ArgumentError, "key '#{key}' already written" if @keys.include?(key)
-
- # rewind by one byte, to overwrite '}'
- file.pos = file.size - 1
-
- file.write(',') if @keys.any?
- file.write(key.to_json)
- file.write(':')
- file.write(value.to_json)
- file.write('}')
-
- @keys.add(key)
- end
-
- def file
- @file ||= File.open(@path, "wb")
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/import_export/json/ndjson_reader.rb b/lib/gitlab/import_export/json/ndjson_reader.rb
index 510da61d3ab..3de56aacf18 100644
--- a/lib/gitlab/import_export/json/ndjson_reader.rb
+++ b/lib/gitlab/import_export/json/ndjson_reader.rb
@@ -17,14 +17,12 @@ module Gitlab
Dir.exist?(@dir_path)
end
- # This can be removed once legacy_reader is deprecated.
- def legacy?
- false
- end
-
def consume_attributes(importable_path)
# This reads from `tree/project.json`
path = file_path("#{importable_path}.json")
+
+ raise Gitlab::ImportExport::Error, 'Invalid file' if !File.exist?(path) || File.symlink?(path)
+
data = File.read(path, MAX_JSON_DOCUMENT_SIZE)
json_decode(data)
end
@@ -36,7 +34,7 @@ module Gitlab
# This reads from `tree/project/merge_requests.ndjson`
path = file_path(importable_path, "#{key}.ndjson")
- next unless File.exist?(path)
+ next if !File.exist?(path) || File.symlink?(path)
File.foreach(path, MAX_JSON_DOCUMENT_SIZE).with_index do |line, line_num|
documents << [json_decode(line), line_num]
diff --git a/lib/gitlab/import_export/json/streaming_serializer.rb b/lib/gitlab/import_export/json/streaming_serializer.rb
index 389ab8b4c97..9bb0770dc90 100644
--- a/lib/gitlab/import_export/json/streaming_serializer.rb
+++ b/lib/gitlab/import_export/json/streaming_serializer.rb
@@ -8,6 +8,8 @@ module Gitlab
BATCH_SIZE = 100
+ attr_reader :exported_objects_count
+
class Raw < String
def to_json(*_args)
to_s
@@ -21,6 +23,7 @@ module Gitlab
@relations_schema = relations_schema
@json_writer = json_writer
@logger = logger
+ @exported_objects_count = 0
end
def execute
@@ -40,21 +43,28 @@ module Gitlab
relations_schema.merge(include: nil, preloads: nil, unsafe: true))
json_writer.write_attributes(exportable_path, attributes)
+
+ increment_exported_objects_counter
end
- def serialize_relation(definition)
+ def serialize_relation(definition, options = {})
raise ArgumentError, 'definition needs to be Hash' unless definition.is_a?(Hash)
raise ArgumentError, 'definition needs to have exactly one Hash element' unless definition.one?
- key, options = definition.first
+ key, definition_options = definition.first
record = exportable.public_send(key) # rubocop: disable GitlabSecurity/PublicSend
+
+ if options[:batch_ids]
+ record = record.where(record.model.primary_key => Array.wrap(options[:batch_ids]).map(&:to_i))
+ end
+
if record.is_a?(ActiveRecord::Relation)
- serialize_many_relations(key, record, options)
+ serialize_many_relations(key, record, definition_options)
elsif record.respond_to?(:each) # this is to support `project_members` that return an Array
- serialize_many_each(key, record, options)
+ serialize_many_each(key, record, definition_options)
else
- serialize_single_relation(key, record, options)
+ serialize_single_relation(key, record, definition_options)
end
end
@@ -76,6 +86,8 @@ module Gitlab
items << exportable_json_record(record, options, key)
+ increment_exported_objects_counter
+
after_read_callback(record)
end
end
@@ -175,6 +187,8 @@ module Gitlab
enumerator = Enumerator.new do |items|
records.each do |record|
items << exportable_json_record(record, options, key)
+
+ increment_exported_objects_counter
end
end
@@ -187,6 +201,8 @@ module Gitlab
json = exportable_json_record(record, options, key)
json_writer.write_relation(@exportable_path, key, json)
+
+ increment_exported_objects_counter
end
def includes
@@ -263,6 +279,10 @@ module Gitlab
message += ". Number of records to export: #{size}" if size
logger.info(message: message, **log_base_data)
end
+
+ def increment_exported_objects_counter
+ @exported_objects_count += 1
+ end
end
end
end
diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml
index d97ffee8698..36a3c73271b 100644
--- a/lib/gitlab/import_export/project/import_export.yml
+++ b/lib/gitlab/import_export/project/import_export.yml
@@ -89,13 +89,15 @@ tree:
- :milestone
- :resource_state_events
- :external_pull_requests
+ - commit_notes:
+ - :author
+ - events:
+ - :push_event_payload
- ci_pipelines:
- - notes:
- - :author
- - events:
- - :push_event_payload
- stages:
- - :statuses
+ - :builds
+ - :generic_commit_statuses
+ - :bridges
- :external_pull_request
- :merge_request
- :pipeline_metadata
@@ -119,9 +121,25 @@ tree:
- label:
- :priorities
- :service_desk_setting
+ - :design_management_repository
group_members:
- :user
+# Used to support old exports that were exported before the removal/rename of the associations
+#
+# For example, statuses of ci_pipelines are no longer exported, and instead, statuses are
+# exported as builds, generic_commit_statuses, and bridges. So in order to allow statuses
+# to be still imported, it is added to the list below.
+import_only_tree:
+ project:
+ - ci_pipelines:
+ - notes:
+ - :author
+ - events:
+ - :push_event_payload
+ - stages:
+ - :statuses
+
# Only include the following attributes for the models specified.
included_attributes:
user:
@@ -529,7 +547,7 @@ included_attributes:
- :source_sha
- :target_sha
external_pull_requests: *external_pull_request_definition
- statuses:
+ statuses: &statuses_definition
- :project_id
- :status
- :finished_at
@@ -562,6 +580,9 @@ included_attributes:
- :scheduled_at
- :scheduling_type
- :ci_stage
+ builds: *statuses_definition
+ generic_commit_statuses: *statuses_definition
+ bridges: *statuses_definition
ci_pipelines:
- :ref
- :sha
@@ -596,6 +617,7 @@ included_attributes:
- :project_id
- :created_at
- :updated_at
+ # - :statuses # old exports use statuses instead of builds, generic_commit_statuses and bridges
actions:
- :event
design: &design_definition
@@ -896,7 +918,7 @@ excluded_attributes:
merge_requests: *merge_request_excluded_definition
award_emoji:
- :awardable_id
- statuses:
+ statuses: &statuses_excluded_definition
- :trace
- :token
- :token_encrypted
@@ -918,6 +940,9 @@ excluded_attributes:
- :processed
- :id_convert_to_bigint
- :stage_id_convert_to_bigint
+ builds: *statuses_excluded_definition
+ generic_commit_statuses: *statuses_excluded_definition
+ bridges: *statuses_excluded_definition
sentry_issue:
- :issue_id
push_event_payload:
@@ -955,6 +980,9 @@ excluded_attributes:
notes:
- :noteable_id
- :review_id
+ commit_notes:
+ - :noteable_id
+ - :review_id
label_links:
- :label_id
- :target_id
@@ -1079,6 +1107,8 @@ methods:
- :squash_option
notes:
- :type
+ commit_notes:
+ - :type
labels:
- :type
label:
@@ -1100,8 +1130,6 @@ methods:
- :type
lists:
- :list_type
- ci_pipelines:
- - :notes
issues:
- :state
@@ -1111,10 +1139,11 @@ methods:
preloads:
issues:
project: :route
- statuses:
- # TODO: We cannot preload tags, as they are not part of `GenericCommitStatus`
- # tags: # needed by tag_list
- project: # deprecated: needed by coverage_regex of Ci::Build
+ builds:
+ metadata:
+ project:
+ bridges:
+ metadata:
merge_requests:
source_project: :route # needed by source_branch_sha and diff_head_sha
target_project: :route # needed by target_branch_sha
@@ -1167,6 +1196,9 @@ ee:
- :milestone
- lists:
- :milestone
+ - approval_rules:
+ - :approval_project_rules_protected_branches
+ - :approval_project_rules_users
included_attributes:
issuable_sla:
@@ -1232,9 +1264,30 @@ ee:
- :description
iterations_cadence:
- :title
+ approval_rules:
+ - :approvals_required
+ - :name
+ - :rule_type
+ - :scanners
+ - :vulnerabilities_allowed
+ - :severity_levels
+ - :report_type
+ - :vulnerability_states
+ - :orchestration_policy_idx
+ - :applies_to_all_protected_branches
+ approval_project_rules_protected_branches:
+ - :protected_branch
+ approval_project_rules_users:
+ - :user_id
excluded_attributes:
project:
- :vulnerability_hooks_integrations
+ approval_rules:
+ - :created_at
+ - :updated_at
+ methods:
+ approval_project_rules_protected_branches:
+ - :branch_name
preloads:
issues:
epic:
diff --git a/lib/gitlab/import_export/project/object_builder.rb b/lib/gitlab/import_export/project/object_builder.rb
index 50a67a746f8..ac28ae6bfe0 100644
--- a/lib/gitlab/import_export/project/object_builder.rb
+++ b/lib/gitlab/import_export/project/object_builder.rb
@@ -60,10 +60,11 @@ module Gitlab
def prepare_attributes
attributes.dup.tap do |atts|
- atts.delete('group') unless epic? || iteration?
+ atts.delete('group') unless group_level_object?
if label?
atts['type'] = 'ProjectLabel' # Always create project labels
+ atts.delete('group_id')
elsif milestone?
if atts['group_id'] # Transform new group milestones into project ones
atts['iid'] = nil
@@ -141,10 +142,6 @@ module Gitlab
klass == MergeRequestDiffCommit
end
- def iteration?
- klass == Iteration
- end
-
# If an existing group milestone used the IID
# claim the IID back and set the group milestone to use one available
# This is necessary to fix situations like the following:
@@ -163,7 +160,11 @@ module Gitlab
end
def group_relation_without_group?
- (epic? || iteration?) && group.nil?
+ group_level_object? && group.nil?
+ end
+
+ def group_level_object?
+ epic?
end
end
end
diff --git a/lib/gitlab/import_export/project/relation_factory.rb b/lib/gitlab/import_export/project/relation_factory.rb
index 4134c428500..5d7e3ea9ed7 100644
--- a/lib/gitlab/import_export/project/relation_factory.rb
+++ b/lib/gitlab/import_export/project/relation_factory.rb
@@ -5,6 +5,7 @@ module Gitlab
module Project
class RelationFactory < Base::RelationFactory
OVERRIDES = { snippets: :project_snippets,
+ commit_notes: 'Note',
ci_pipelines: 'Ci::Pipeline',
pipelines: 'Ci::Pipeline',
stages: 'Ci::Stage',
@@ -12,6 +13,7 @@ module Gitlab
triggers: 'Ci::Trigger',
pipeline_schedules: 'Ci::PipelineSchedule',
builds: 'Ci::Build',
+ bridges: 'Ci::Bridge',
runners: 'Ci::Runner',
pipeline_metadata: 'Ci::PipelineMetadata',
hooks: 'ProjectHook',
@@ -20,6 +22,7 @@ module Gitlab
create_access_levels: 'ProtectedTag::CreateAccessLevel',
design: 'DesignManagement::Design',
designs: 'DesignManagement::Design',
+ design_management_repository: 'DesignManagement::Repository',
design_versions: 'DesignManagement::Version',
actions: 'DesignManagement::Action',
labels: :project_labels,
@@ -37,7 +40,7 @@ module Gitlab
committer: 'MergeRequest::DiffCommitUser',
merge_request_diff_commits: 'MergeRequestDiffCommit' }.freeze
- BUILD_MODELS = %i[Ci::Build commit_status].freeze
+ BUILD_MODELS = %i[Ci::Build Ci::Bridge commit_status generic_commit_status].freeze
GROUP_REFERENCES = %w[group_id].freeze
@@ -83,13 +86,14 @@ module Gitlab
def setup_models
case @relation_name
when :merge_request_diff_files then setup_diff
- when :notes then setup_note
+ when :notes, :Note then setup_note
when :'Ci::Pipeline' then setup_pipeline
when *BUILD_MODELS then setup_build
when :issues then setup_issue
when :'Ci::PipelineSchedule' then setup_pipeline_schedule
when :'ProtectedBranch::MergeAccessLevel' then setup_protected_branch_access_level
when :'ProtectedBranch::PushAccessLevel' then setup_protected_branch_access_level
+ when :ApprovalProjectRulesProtectedBranch then setup_merge_approval_protected_branch
when :releases then setup_release
end
@@ -142,9 +146,22 @@ module Gitlab
def setup_pipeline
@relation_hash.fetch('stages', []).each do |stage|
+ # old export files have statuses
stage.statuses.each do |status|
status.pipeline = imported_object
end
+
+ stage.builds.each do |status|
+ status.pipeline = imported_object
+ end
+
+ stage.bridges.each do |status|
+ status.pipeline = imported_object
+ end
+
+ stage.generic_commit_statuses.each do |status|
+ status.pipeline = imported_object
+ end
end
end
@@ -180,6 +197,13 @@ module Gitlab
root_ancestor.max_member_access_for_user(@user) == Gitlab::Access::OWNER
end
+ def setup_merge_approval_protected_branch
+ source_branch_name = @relation_hash.delete('branch_name')
+ target_branch = @importable.protected_branches.find_by(name: source_branch_name)
+
+ @relation_hash['protected_branch'] = target_branch
+ end
+
def compute_relative_position
return unless max_relative_position
diff --git a/lib/gitlab/import_export/project/relation_tree_restorer.rb b/lib/gitlab/import_export/project/relation_tree_restorer.rb
index 47196db6f8a..b5247754199 100644
--- a/lib/gitlab/import_export/project/relation_tree_restorer.rb
+++ b/lib/gitlab/import_export/project/relation_tree_restorer.rb
@@ -5,10 +5,14 @@ module Gitlab
module Project
class RelationTreeRestorer < ImportExport::Group::RelationTreeRestorer
# Relations which cannot be saved at project level (and have a group assigned)
- GROUP_MODELS = [GroupLabel, Milestone, Epic, Iteration].freeze
+ GROUP_MODELS = [GroupLabel, Milestone, Epic].freeze
private
+ def group_models
+ GROUP_MODELS
+ end
+
def bulk_insert_enabled
true
end
@@ -19,9 +23,11 @@ module Gitlab
end
def relation_invalid_for_importable?(relation_object)
- GROUP_MODELS.include?(relation_object.class) && relation_object.group_id
+ group_models.include?(relation_object.class) && relation_object.group_id
end
end
end
end
end
+
+Gitlab::ImportExport::Project::RelationTreeRestorer.prepend_mod
diff --git a/lib/gitlab/import_export/project/sample/relation_tree_restorer.rb b/lib/gitlab/import_export/project/sample/relation_tree_restorer.rb
index 034122a9f14..639f34980ff 100644
--- a/lib/gitlab/import_export/project/sample/relation_tree_restorer.rb
+++ b/lib/gitlab/import_export/project/sample/relation_tree_restorer.rb
@@ -18,8 +18,6 @@ module Gitlab
end
def dates
- return [] if @relation_reader.legacy?
-
RelationFactory::DATE_MODELS.flat_map do |tag|
@relation_reader.consume_relation(@importable_path, tag, mark_as_consumed: false).map do |model|
model.first['due_date']
diff --git a/lib/gitlab/import_export/project/tree_restorer.rb b/lib/gitlab/import_export/project/tree_restorer.rb
index 47f82a901b7..e791424875a 100644
--- a/lib/gitlab/import_export/project/tree_restorer.rb
+++ b/lib/gitlab/import_export/project/tree_restorer.rb
@@ -17,7 +17,7 @@ module Gitlab
end
def restore
- unless relation_reader
+ unless relation_reader.exist?
raise Gitlab::ImportExport::Error, 'invalid import format'
end
@@ -47,28 +47,11 @@ module Gitlab
private
def relation_reader
- strong_memoize(:relation_reader) do
- [ndjson_relation_reader, legacy_relation_reader]
- .compact.find(&:exist?)
- end
- end
-
- def ndjson_relation_reader
- return unless Feature.enabled?(:project_import_ndjson, project.namespace)
-
- ImportExport::Json::NdjsonReader.new(
+ @relation_reader ||= ImportExport::Json::NdjsonReader.new(
File.join(shared.export_path, 'tree')
)
end
- def legacy_relation_reader
- ImportExport::Json::LegacyReader::File.new(
- File.join(shared.export_path, 'project.json'),
- relation_names: reader.project_relation_names,
- allowed_path: importable_path
- )
- end
-
def relation_tree_restorer
@relation_tree_restorer ||= relation_tree_restorer_class.new(
user: @user,
diff --git a/lib/gitlab/import_export/project/tree_saver.rb b/lib/gitlab/import_export/project/tree_saver.rb
index 05b96f7e8ce..fd5fa73764e 100644
--- a/lib/gitlab/import_export/project/tree_saver.rb
+++ b/lib/gitlab/import_export/project/tree_saver.rb
@@ -81,13 +81,10 @@ module Gitlab
end
def json_writer
- @json_writer ||= if ::Feature.enabled?(:project_export_as_ndjson, @project.namespace)
- full_path = File.join(@shared.export_path, 'tree')
- Gitlab::ImportExport::Json::NdjsonWriter.new(full_path)
- else
- full_path = File.join(@shared.export_path, ImportExport.project_filename)
- Gitlab::ImportExport::Json::LegacyWriter.new(full_path, allowed_path: 'project')
- end
+ @json_writer ||= begin
+ full_path = File.join(@shared.export_path, 'tree')
+ Gitlab::ImportExport::Json::NdjsonWriter.new(full_path)
+ end
end
end
end
diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb
index a94ea6f595b..77b0df765c4 100644
--- a/lib/gitlab/import_sources.rb
+++ b/lib/gitlab/import_sources.rb
@@ -9,18 +9,15 @@ module Gitlab
module ImportSources
ImportSource = Struct.new(:name, :title, :importer)
- # We exclude `bare_repository` here as it has no import class associated
IMPORT_TABLE = [
ImportSource.new('github', 'GitHub', Gitlab::GithubImport::ParallelImporter),
ImportSource.new('bitbucket', 'Bitbucket Cloud', Gitlab::BitbucketImport::Importer),
ImportSource.new('bitbucket_server', 'Bitbucket Server', Gitlab::BitbucketServerImport::Importer),
- ImportSource.new('gitlab', 'GitLab.com', Gitlab::GitlabImport::Importer),
ImportSource.new('fogbugz', 'FogBugz', Gitlab::FogbugzImport::Importer),
ImportSource.new('git', 'Repository by URL', nil),
ImportSource.new('gitlab_project', 'GitLab export', Gitlab::ImportExport::Importer),
ImportSource.new('gitea', 'Gitea', Gitlab::LegacyGithubImport::Importer),
- ImportSource.new('manifest', 'Manifest file', nil),
- ImportSource.new('phabricator', 'Phabricator', Gitlab::PhabricatorImport::Importer)
+ ImportSource.new('manifest', 'Manifest file', nil)
].freeze
class << self
diff --git a/lib/gitlab/incoming_email.rb b/lib/gitlab/incoming_email.rb
deleted file mode 100644
index d34c19bc9fc..00000000000
--- a/lib/gitlab/incoming_email.rb
+++ /dev/null
@@ -1,34 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module IncomingEmail
- class << self
- include Gitlab::Email::Common
-
- def config
- incoming_email_config
- end
-
- def key_from_address(address, wildcard_address: nil)
- wildcard_address ||= config.address
- regex = address_regex(wildcard_address)
- return unless regex
-
- match = address.match(regex)
- return unless match
-
- match[1]
- end
-
- private
-
- def address_regex(wildcard_address)
- return unless wildcard_address
-
- regex = Regexp.escape(wildcard_address)
- regex = regex.sub(Regexp.escape(WILDCARD_PLACEHOLDER), '(.+)')
- Regexp.new(/\A<?#{regex}>?\z/).freeze
- end
- end
- end
-end
diff --git a/lib/gitlab/instrumentation/redis.rb b/lib/gitlab/instrumentation/redis.rb
index a664656c467..590153ad9cd 100644
--- a/lib/gitlab/instrumentation/redis.rb
+++ b/lib/gitlab/instrumentation/redis.rb
@@ -19,8 +19,8 @@ module Gitlab
end << ActionCable
).freeze
- # Milliseconds represented in seconds (from 1 millisecond to 2 seconds).
- QUERY_TIME_BUCKETS = [0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2].freeze
+ # Milliseconds represented in seconds
+ QUERY_TIME_BUCKETS = [0.1, 0.25, 0.5].freeze
class << self
include ::Gitlab::Instrumentation::RedisPayload
diff --git a/lib/gitlab/instrumentation/redis_base.rb b/lib/gitlab/instrumentation/redis_base.rb
index de24132a28e..00a7387afe2 100644
--- a/lib/gitlab/instrumentation/redis_base.rb
+++ b/lib/gitlab/instrumentation/redis_base.rb
@@ -118,6 +118,14 @@ module Gitlab
@exception_counter.increment({ storage: storage_key, exception: ex.class.to_s })
end
+ def instance_count_cluster_redirection(ex)
+ # This metric is meant to give a client side view of how often are commands
+ # redirected to the right node, especially during resharding..
+ # This metric can be used for Redis alerting and service health monitoring.
+ @redirection_counter ||= Gitlab::Metrics.counter(:gitlab_redis_client_redirections_total, 'Client side Redis Cluster redirection count, per Redis node, per slot')
+ @redirection_counter.increment(decompose_redirection_message(ex.message).merge({ storage: storage_key }))
+ end
+
def instance_observe_duration(duration)
@request_latency_histogram ||= Gitlab::Metrics.histogram(
:gitlab_redis_client_requests_duration_seconds,
@@ -129,6 +137,10 @@ module Gitlab
@request_latency_histogram.observe({ storage: storage_key }, duration)
end
+ def log_exception(ex)
+ ::Gitlab::ErrorTracking.log_exception(ex, storage: storage_key)
+ end
+
private
def request_count_key
@@ -162,6 +174,11 @@ module Gitlab
def build_key(namespace)
"#{storage_key}_#{namespace}"
end
+
+ def decompose_redirection_message(err_msg)
+ redirection_type, _, target_node_key = err_msg.split
+ { redirection_type: redirection_type, target_node_key: target_node_key }
+ end
end
end
end
diff --git a/lib/gitlab/instrumentation/redis_interceptor.rb b/lib/gitlab/instrumentation/redis_interceptor.rb
index 35dd7cbfeb8..b3fbe30e583 100644
--- a/lib/gitlab/instrumentation/redis_interceptor.rb
+++ b/lib/gitlab/instrumentation/redis_interceptor.rb
@@ -3,7 +3,7 @@
module Gitlab
module Instrumentation
module RedisInterceptor
- APDEX_EXCLUDE = %w[brpop blpop brpoplpush bzpopmin bzpopmax xread xreadgroup].freeze
+ APDEX_EXCLUDE = %w[brpop blpop brpoplpush bzpopmin bzpopmax command xread xreadgroup].freeze
def call(command)
instrument_call([command]) do
@@ -40,7 +40,13 @@ module Gitlab
yield
rescue ::Redis::BaseError => ex
- instrumentation_class.instance_count_exception(ex)
+ if ex.message.start_with?('MOVED', 'ASK')
+ instrumentation_class.instance_count_cluster_redirection(ex)
+ else
+ instrumentation_class.instance_count_exception(ex)
+ end
+
+ instrumentation_class.log_exception(ex)
raise ex
ensure
duration = Gitlab::Metrics::System.monotonic_time - start
diff --git a/lib/gitlab/instrumentation/zoekt.rb b/lib/gitlab/instrumentation/zoekt.rb
new file mode 100644
index 00000000000..cd9b15bcee8
--- /dev/null
+++ b/lib/gitlab/instrumentation/zoekt.rb
@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Instrumentation
+ class Zoekt
+ ZOEKT_REQUEST_COUNT = :zoekt_request_count
+ ZOEKT_CALL_DURATION = :zoekt_call_duration
+ ZOEKT_CALL_DETAILS = :zoekt_call_details
+
+ class << self
+ def get_request_count
+ ::Gitlab::SafeRequestStore[ZOEKT_REQUEST_COUNT] || 0
+ end
+
+ def increment_request_count
+ ::Gitlab::SafeRequestStore[ZOEKT_REQUEST_COUNT] ||= 0
+ ::Gitlab::SafeRequestStore[ZOEKT_REQUEST_COUNT] += 1
+ end
+
+ def detail_store
+ ::Gitlab::SafeRequestStore[ZOEKT_CALL_DETAILS] ||= []
+ end
+
+ def query_time
+ query_time = ::Gitlab::SafeRequestStore[ZOEKT_CALL_DURATION] || 0
+ query_time.round(::Gitlab::InstrumentationHelper::DURATION_PRECISION)
+ end
+
+ def add_duration(duration)
+ ::Gitlab::SafeRequestStore[ZOEKT_CALL_DURATION] ||= 0
+ ::Gitlab::SafeRequestStore[ZOEKT_CALL_DURATION] += duration
+ end
+
+ def add_call_details(duration:, method:, path:, params: nil, body: nil)
+ return unless Gitlab::PerformanceBar.enabled_for_request?
+
+ detail_store << {
+ method: method,
+ path: path,
+ params: params,
+ body: body,
+ duration: duration,
+ backtrace: ::Gitlab::BacktraceCleaner.clean_backtrace(caller)
+ }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/instrumentation_helper.rb b/lib/gitlab/instrumentation_helper.rb
index 15a760fada0..2a3c4db5ffa 100644
--- a/lib/gitlab/instrumentation_helper.rb
+++ b/lib/gitlab/instrumentation_helper.rb
@@ -6,16 +6,8 @@ module Gitlab
DURATION_PRECISION = 6 # microseconds
- def init_instrumentation_data(request_ip: nil)
- # Set `request_start_time` only if this is request
- # This is done, as `request_start_time` imply `request_deadline`
- if request_ip
- Gitlab::RequestContext.instance.client_ip = request_ip
- Gitlab::RequestContext.instance.request_start_time = Gitlab::Metrics::System.real_time
- end
-
- Gitlab::RequestContext.instance.start_thread_cpu_time = Gitlab::Metrics::System.thread_cpu_time
- Gitlab::RequestContext.instance.thread_memory_allocations = Gitlab::Memory::Instrumentation.start_thread_memory_allocations
+ def init_instrumentation_data
+ Gitlab::RequestContext.start_thread_context
end
def add_instrumentation_data(payload)
@@ -23,6 +15,7 @@ module Gitlab
instrument_rugged(payload)
instrument_redis(payload)
instrument_elasticsearch(payload)
+ instrument_zoekt(payload)
instrument_throttle(payload)
instrument_active_record(payload)
instrument_external_http(payload)
@@ -72,6 +65,17 @@ module Gitlab
payload[:elasticsearch_timed_out_count] = Gitlab::Instrumentation::ElasticsearchTransport.get_timed_out_count
end
+ def instrument_zoekt(payload)
+ # Zoekt integration is only available in EE but instrumentation
+ # only depends on the Gem which is also available in FOSS.
+ zoekt_calls = Gitlab::Instrumentation::Zoekt.get_request_count
+
+ return if zoekt_calls == 0
+
+ payload[:zoekt_calls] = zoekt_calls
+ payload[:zoekt_duration_s] = Gitlab::Instrumentation::Zoekt.query_time
+ end
+
def instrument_external_http(payload)
external_http_count = Gitlab::Metrics::Subscribers::ExternalHttp.request_count
diff --git a/lib/gitlab/issuable/clone/copy_resource_events_service.rb b/lib/gitlab/issuable/clone/copy_resource_events_service.rb
index 448ac4c2ae0..22b0d5c4fb2 100644
--- a/lib/gitlab/issuable/clone/copy_resource_events_service.rb
+++ b/lib/gitlab/issuable/clone/copy_resource_events_service.rb
@@ -69,9 +69,9 @@ module Gitlab
def copy_events(table_name, events_to_copy)
events_to_copy.find_in_batches do |batch|
- events = batch.map do |event|
+ events = batch.filter_map do |event|
yield(event)
- end.compact
+ end
ApplicationRecord.legacy_bulk_insert(table_name, events) # rubocop:disable Gitlab/BulkInsert
end
diff --git a/lib/gitlab/issues/rebalancing/state.rb b/lib/gitlab/issues/rebalancing/state.rb
index 36346564b39..f1f6cc55a2b 100644
--- a/lib/gitlab/issues/rebalancing/state.rb
+++ b/lib/gitlab/issues/rebalancing/state.rb
@@ -38,10 +38,10 @@ module Gitlab
def rebalance_in_progress?
is_running = case rebalanced_container_type
when NAMESPACE
- namespace_ids = self.class.current_rebalancing_containers.map { |string| string.split("#{NAMESPACE}/").second.to_i }.compact
+ namespace_ids = self.class.current_rebalancing_containers.filter_map { |string| string.split("#{NAMESPACE}/").second.to_i }
namespace_ids.include?(root_namespace.id)
when PROJECT
- project_ids = self.class.current_rebalancing_containers.map { |string| string.split("#{PROJECT}/").second.to_i }.compact
+ project_ids = self.class.current_rebalancing_containers.filter_map { |string| string.split("#{PROJECT}/").second.to_i }
project_ids.include?(projects.take.id) # rubocop:disable CodeReuse/ActiveRecord
else
false
diff --git a/lib/gitlab/jira_import/issues_importer.rb b/lib/gitlab/jira_import/issues_importer.rb
index 7b031c26b72..458f7c3f470 100644
--- a/lib/gitlab/jira_import/issues_importer.rb
+++ b/lib/gitlab/jira_import/issues_importer.rb
@@ -48,7 +48,7 @@ module Gitlab
end
def schedule_issue_import_workers(issues)
- next_iid = Issue.with_project_iid_supply(project, &:next_value)
+ next_iid = Issue.with_namespace_iid_supply(project.project_namespace, &:next_value)
issues.each do |jira_issue|
# Technically it's possible that the same work is performed multiple
@@ -71,7 +71,7 @@ module Gitlab
job_waiter.jobs_remaining += 1
- next_iid = Issue.with_project_iid_supply(project, &:next_value)
+ next_iid = Issue.with_namespace_iid_supply(project.project_namespace, &:next_value)
# Mark the issue as imported immediately so we don't end up
# importing it multiple times within same import.
diff --git a/lib/gitlab/jira_import/metadata_collector.rb b/lib/gitlab/jira_import/metadata_collector.rb
index 4551f38ba98..090b95ac14a 100644
--- a/lib/gitlab/jira_import/metadata_collector.rb
+++ b/lib/gitlab/jira_import/metadata_collector.rb
@@ -45,7 +45,7 @@ module Gitlab
def add_versions
return if fields['fixVersions'].blank? || !fields['fixVersions'].is_a?(Array)
- versions = fields['fixVersions'].map { |version| version['name'] }.compact.join(', ')
+ versions = fields['fixVersions'].filter_map { |version| version['name'] }.join(', ')
metadata << "- Fix versions: #{versions}"
end
diff --git a/lib/gitlab/json.rb b/lib/gitlab/json.rb
index bdfbe2041cd..e515d00f8d8 100644
--- a/lib/gitlab/json.rb
+++ b/lib/gitlab/json.rb
@@ -167,7 +167,7 @@ module Gitlab
# @return [Boolean, String, Array, Hash, Object]
# @raise [JSON::ParserError]
def handle_legacy_mode!(data)
- return data unless Feature.feature_flags_available?
+ return data unless Feature::FlipperFeature.table_exists?
return data unless Feature.enabled?(:json_wrapper_legacy_mode)
raise parser_error if INVALID_LEGACY_TYPES.any? { |type| data.is_a?(type) }
diff --git a/lib/gitlab/json_cache.rb b/lib/gitlab/json_cache.rb
index d2916a01809..0087c2accc3 100644
--- a/lib/gitlab/json_cache.rb
+++ b/lib/gitlab/json_cache.rb
@@ -112,7 +112,7 @@ module Gitlab
end
def parse_entries(values, klass)
- values.map { |value| parse_entry(value, klass) }.compact
+ values.filter_map { |value| parse_entry(value, klass) }
end
end
end
diff --git a/lib/gitlab/kas/client.rb b/lib/gitlab/kas/client.rb
index 768810d5545..43546d04087 100644
--- a/lib/gitlab/kas/client.rb
+++ b/lib/gitlab/kas/client.rb
@@ -8,7 +8,8 @@ module Gitlab
STUB_CLASSES = {
agent_tracker: Gitlab::Agent::AgentTracker::Rpc::AgentTracker::Stub,
- configuration_project: Gitlab::Agent::ConfigurationProject::Rpc::ConfigurationProject::Stub
+ configuration_project: Gitlab::Agent::ConfigurationProject::Rpc::ConfigurationProject::Stub,
+ notifications: Gitlab::Agent::Notifications::Rpc::Notifications::Stub
}.freeze
ConfigurationError = Class.new(StandardError)
@@ -39,6 +40,18 @@ module Gitlab
.to_a
end
+ def send_git_push_event(project:)
+ request = Gitlab::Agent::Notifications::Rpc::GitPushEventRequest.new(
+ project: Gitlab::Agent::Notifications::Rpc::Project.new(
+ id: project.id,
+ full_path: project.full_path
+ )
+ )
+
+ stub_for(:notifications)
+ .git_push_event(request, metadata: metadata)
+ end
+
private
def stub_for(service)
diff --git a/lib/gitlab/kas/user_access.rb b/lib/gitlab/kas/user_access.rb
new file mode 100644
index 00000000000..65ae399d826
--- /dev/null
+++ b/lib/gitlab/kas/user_access.rb
@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Kas
+ # The name of the cookie that will be used for the KAS cookie
+ COOKIE_KEY = '_gitlab_kas'
+ DEFAULT_ENCRYPTED_COOKIE_CIPHER = 'aes-256-gcm'
+
+ class UserAccess
+ class << self
+ def enabled?
+ ::Gitlab::Kas.enabled? && ::Feature.enabled?(:kas_user_access)
+ end
+
+ def enabled_for?(agent)
+ enabled? && ::Feature.enabled?(:kas_user_access_project, agent.project)
+ end
+
+ def encrypt_public_session_id(data)
+ encryptor.encrypt_and_sign(data.to_json, purpose: public_session_id_purpose)
+ end
+
+ def decrypt_public_session_id(data)
+ decrypted = encryptor.decrypt_and_verify(data, purpose: public_session_id_purpose)
+ ::Gitlab::Json.parse(decrypted)
+ end
+
+ def valid_authenticity_token?(session, masked_authenticity_token)
+ # rubocop:disable GitlabSecurity/PublicSend
+ ActionController::Base.new.send(:valid_authenticity_token?, session, masked_authenticity_token)
+ # rubocop:enable GitlabSecurity/PublicSend
+ end
+
+ def cookie_data(public_session_id)
+ uri = URI(::Gitlab::Kas.tunnel_url)
+
+ cookie = {
+ value: encrypt_public_session_id(public_session_id),
+ expires: 1.day,
+ httponly: true,
+ path: uri.path.presence || '/',
+ secure: Gitlab.config.gitlab.https
+ }
+ # Only set domain attribute if KAS is on a subdomain.
+ # When on the same domain, we can omit the attribute.
+ gitlab_host = Gitlab.config.gitlab.host
+ cookie[:domain] = gitlab_host if uri.host.end_with?(".#{gitlab_host}")
+
+ cookie
+ end
+
+ private
+
+ def encryptor
+ action_dispatch_config = Gitlab::Application.config.action_dispatch
+ serializer = ActiveSupport::MessageEncryptor::NullSerializer
+ key_generator = ::Gitlab::Application.key_generator
+
+ cipher = action_dispatch_config.encrypted_cookie_cipher || DEFAULT_ENCRYPTED_COOKIE_CIPHER
+ salt = action_dispatch_config.authenticated_encrypted_cookie_salt
+ key_len = ActiveSupport::MessageEncryptor.key_len(cipher)
+ secret = key_generator.generate_key(salt, key_len)
+
+ ActiveSupport::MessageEncryptor.new(secret, cipher: cipher, serializer: serializer)
+ end
+
+ def public_session_id_purpose
+ "kas.user_public_session_id"
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/kubernetes/helm/api.rb b/lib/gitlab/kubernetes/helm/api.rb
deleted file mode 100644
index ceda18442d6..00000000000
--- a/lib/gitlab/kubernetes/helm/api.rb
+++ /dev/null
@@ -1,126 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Kubernetes
- module Helm
- class API
- def initialize(kubeclient)
- @kubeclient = kubeclient
- @namespace = Gitlab::Kubernetes::Namespace.new(
- Gitlab::Kubernetes::Helm::NAMESPACE,
- kubeclient,
- labels: Gitlab::Kubernetes::Helm::NAMESPACE_LABELS
- )
- end
-
- def install(command)
- namespace.ensure_exists!
-
- create_service_account(command)
- create_cluster_role_binding(command)
- create_config_map(command)
-
- delete_pod!(command.pod_name)
- kubeclient.create_pod(command.pod_resource)
- end
-
- alias_method :update, :install
-
- def uninstall(command)
- namespace.ensure_exists!
- create_config_map(command)
-
- delete_pod!(command.pod_name)
- kubeclient.create_pod(command.pod_resource)
- end
-
- ##
- # Returns Pod phase
- #
- # https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase
- #
- # values: "Pending", "Running", "Succeeded", "Failed", "Unknown"
- #
- def status(pod_name)
- kubeclient.get_pod(pod_name, namespace.name).status.phase
- end
-
- def log(pod_name)
- kubeclient.get_pod_log(pod_name, namespace.name).body
- end
-
- def delete_pod!(pod_name)
- kubeclient.delete_pod(pod_name, namespace.name)
- rescue ::Kubeclient::ResourceNotFoundError
- # no-op
- end
-
- def get_config_map(config_map_name)
- namespace.ensure_exists!
-
- kubeclient.get_config_map(config_map_name, namespace.name)
- end
-
- private
-
- attr_reader :kubeclient, :namespace
-
- def create_config_map(command)
- command.config_map_resource.tap do |config_map_resource|
- break unless config_map_resource
-
- if config_map_exists?(config_map_resource)
- kubeclient.update_config_map(config_map_resource)
- else
- kubeclient.create_config_map(config_map_resource)
- end
- end
- end
-
- def update_config_map(command)
- command.config_map_resource.tap do |config_map_resource|
- kubeclient.update_config_map(config_map_resource)
- end
- end
-
- def create_service_account(command)
- command.service_account_resource.tap do |service_account_resource|
- break unless service_account_resource
-
- if service_account_exists?(service_account_resource)
- kubeclient.update_service_account(service_account_resource)
- else
- kubeclient.create_service_account(service_account_resource)
- end
- end
- end
-
- def create_cluster_role_binding(command)
- command.cluster_role_binding_resource.tap do |cluster_role_binding_resource|
- break unless cluster_role_binding_resource
-
- kubeclient.update_cluster_role_binding(cluster_role_binding_resource)
- end
- end
-
- def config_map_exists?(resource)
- kubeclient.get_config_map(resource.metadata.name, resource.metadata.namespace)
- rescue ::Kubeclient::ResourceNotFoundError
- false
- end
-
- def service_account_exists?(resource)
- kubeclient.get_service_account(resource.metadata.name, resource.metadata.namespace)
- rescue ::Kubeclient::ResourceNotFoundError
- false
- end
-
- def cluster_role_binding_exists?(resource)
- kubeclient.get_cluster_role_binding(resource.metadata.name)
- rescue ::Kubeclient::ResourceNotFoundError
- false
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/kubernetes/helm/pod.rb b/lib/gitlab/kubernetes/helm/pod.rb
deleted file mode 100644
index 9d0207e6b1f..00000000000
--- a/lib/gitlab/kubernetes/helm/pod.rb
+++ /dev/null
@@ -1,82 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Kubernetes
- module Helm
- class Pod
- def initialize(command, namespace_name, service_account_name: nil)
- @command = command
- @namespace_name = namespace_name
- @service_account_name = service_account_name
- end
-
- def generate
- spec = { containers: [container_specification], restartPolicy: 'Never' }
-
- spec[:volumes] = volumes_specification
- spec[:containers][0][:volumeMounts] = volume_mounts_specification
- spec[:serviceAccountName] = service_account_name if service_account_name
-
- ::Kubeclient::Resource.new(metadata: metadata, spec: spec)
- end
-
- private
-
- attr_reader :command, :namespace_name, :service_account_name
-
- def container_specification
- {
- name: 'helm',
- image: "registry.gitlab.com/gitlab-org/cluster-integration/helm-install-image/releases/#{command.class::HELM_VERSION}-kube-#{Gitlab::Kubernetes::Helm::KUBECTL_VERSION}-alpine-3.12",
- env: generate_pod_env(command),
- command: %w(/bin/sh),
- args: %w(-c $(COMMAND_SCRIPT))
- }
- end
-
- def labels
- {
- 'gitlab.org/action': 'install',
- 'gitlab.org/application': command.name
- }
- end
-
- def metadata
- {
- name: command.pod_name,
- namespace: namespace_name,
- labels: labels
- }
- end
-
- def generate_pod_env(command)
- command.env.merge(
- HELM_VERSION: command.class::HELM_VERSION,
- COMMAND_SCRIPT: command.generate_script
- ).map { |key, value| { name: key, value: value } }
- end
-
- def volumes_specification
- [
- {
- name: 'configuration-volume',
- configMap: {
- name: "values-content-configuration-#{command.name}",
- items: command.file_names.map { |name| { key: name, path: name } }
- }
- }
- ]
- end
-
- def volume_mounts_specification
- [
- {
- name: 'configuration-volume',
- mountPath: "/data/helm/#{command.name}/config"
- }
- ]
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/kubernetes/helm/v2/base_command.rb b/lib/gitlab/kubernetes/helm/v2/base_command.rb
deleted file mode 100644
index 26c77b2149e..00000000000
--- a/lib/gitlab/kubernetes/helm/v2/base_command.rb
+++ /dev/null
@@ -1,93 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Kubernetes
- module Helm
- module V2
- class BaseCommand
- attr_reader :name, :files
-
- HELM_VERSION = '2.17.0'
-
- def initialize(rbac:, name:, files:)
- @rbac = rbac
- @name = name
- @files = files
- end
-
- def env
- { TILLER_NAMESPACE: namespace }
- end
-
- def rbac?
- @rbac
- end
-
- def pod_resource
- pod_service_account_name = rbac? ? service_account_name : nil
-
- Gitlab::Kubernetes::Helm::Pod.new(self, namespace, service_account_name: pod_service_account_name).generate
- end
-
- def generate_script
- <<~HEREDOC
- set -xeo pipefail
- HEREDOC
- end
-
- def pod_name
- "install-#{name}"
- end
-
- def config_map_resource
- Gitlab::Kubernetes::ConfigMap.new(name, files).generate
- end
-
- def service_account_resource
- return unless rbac?
-
- Gitlab::Kubernetes::ServiceAccount.new(service_account_name, namespace).generate
- end
-
- def cluster_role_binding_resource
- return unless rbac?
-
- subjects = [{ kind: 'ServiceAccount', name: service_account_name, namespace: namespace }]
-
- Gitlab::Kubernetes::ClusterRoleBinding.new(
- cluster_role_binding_name,
- cluster_role_name,
- subjects
- ).generate
- end
-
- def file_names
- files.keys
- end
-
- private
-
- def files_dir
- "/data/helm/#{name}/config"
- end
-
- def namespace
- Gitlab::Kubernetes::Helm::NAMESPACE
- end
-
- def service_account_name
- Gitlab::Kubernetes::Helm::SERVICE_ACCOUNT
- end
-
- def cluster_role_binding_name
- Gitlab::Kubernetes::Helm::CLUSTER_ROLE_BINDING
- end
-
- def cluster_role_name
- Gitlab::Kubernetes::Helm::CLUSTER_ROLE
- end
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/kubernetes/helm/v2/certificate.rb b/lib/gitlab/kubernetes/helm/v2/certificate.rb
deleted file mode 100644
index 17ea2eb5188..00000000000
--- a/lib/gitlab/kubernetes/helm/v2/certificate.rb
+++ /dev/null
@@ -1,75 +0,0 @@
-# frozen_string_literal: true
-module Gitlab
- module Kubernetes
- module Helm
- module V2
- class Certificate
- INFINITE_EXPIRY = 1000.years
- SHORT_EXPIRY = 30.minutes
-
- attr_reader :key, :cert
-
- def key_string
- @key.to_s
- end
-
- def cert_string
- @cert.to_pem
- end
-
- def self.from_strings(key_string, cert_string)
- key = OpenSSL::PKey::RSA.new(key_string)
- cert = OpenSSL::X509::Certificate.new(cert_string)
- new(key, cert)
- end
-
- def self.generate_root
- _issue(signed_by: nil, expires_in: INFINITE_EXPIRY, certificate_authority: true)
- end
-
- def issue(expires_in: SHORT_EXPIRY)
- self.class._issue(signed_by: self, expires_in: expires_in, certificate_authority: false)
- end
-
- private
-
- def self._issue(signed_by:, expires_in:, certificate_authority:)
- key = OpenSSL::PKey::RSA.new(4096)
- public_key = key.public_key
-
- subject = OpenSSL::X509::Name.parse("/C=US")
-
- cert = OpenSSL::X509::Certificate.new
- cert.subject = subject
-
- cert.issuer = signed_by&.cert&.subject || subject
-
- cert.not_before = Time.now.utc
- cert.not_after = expires_in.from_now.utc
- cert.public_key = public_key
- cert.serial = 0x0
- cert.version = 2
-
- if certificate_authority
- extension_factory = OpenSSL::X509::ExtensionFactory.new
- extension_factory.subject_certificate = cert
- extension_factory.issuer_certificate = cert
- cert.add_extension(extension_factory.create_extension('subjectKeyIdentifier', 'hash'))
- cert.add_extension(extension_factory.create_extension('basicConstraints', 'CA:TRUE', true))
- cert.add_extension(extension_factory.create_extension('keyUsage', 'cRLSign,keyCertSign', true))
- end
-
- cert.sign(signed_by&.key || key, OpenSSL::Digest.new('SHA256'))
-
- new(key, cert)
- end
-
- def initialize(key, cert)
- @key = key
- @cert = cert
- end
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/kubernetes/helm/v2/client_command.rb b/lib/gitlab/kubernetes/helm/v2/client_command.rb
deleted file mode 100644
index 8b15af9aeea..00000000000
--- a/lib/gitlab/kubernetes/helm/v2/client_command.rb
+++ /dev/null
@@ -1,29 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Kubernetes
- module Helm
- module V2
- module ClientCommand
- def init_command
- <<~SHELL.chomp
- export HELM_HOST="localhost:44134"
- tiller -listen ${HELM_HOST} -alsologtostderr &
- helm init --client-only
- SHELL
- end
-
- def repository_command
- ['helm', 'repo', 'add', name, repository].shelljoin if repository
- end
-
- private
-
- def repository_update_command
- 'helm repo update'
- end
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/kubernetes/helm/v2/delete_command.rb b/lib/gitlab/kubernetes/helm/v2/delete_command.rb
deleted file mode 100644
index 4d52fc1398f..00000000000
--- a/lib/gitlab/kubernetes/helm/v2/delete_command.rb
+++ /dev/null
@@ -1,38 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Kubernetes
- module Helm
- module V2
- class DeleteCommand < BaseCommand
- include ClientCommand
-
- attr_reader :predelete, :postdelete
-
- def initialize(predelete: nil, postdelete: nil, **args)
- super(**args)
- @predelete = predelete
- @postdelete = postdelete
- end
-
- def generate_script
- super + [
- init_command,
- predelete,
- delete_command,
- postdelete
- ].compact.join("\n")
- end
-
- def pod_name
- "uninstall-#{name}"
- end
-
- def delete_command
- ['helm', 'delete', '--purge', name].shelljoin
- end
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/kubernetes/helm/v2/init_command.rb b/lib/gitlab/kubernetes/helm/v2/init_command.rb
deleted file mode 100644
index f8b52feb5b6..00000000000
--- a/lib/gitlab/kubernetes/helm/v2/init_command.rb
+++ /dev/null
@@ -1,45 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Kubernetes
- module Helm
- module V2
- class InitCommand < BaseCommand
- def generate_script
- super + [
- init_helm_command
- ].join("\n")
- end
-
- private
-
- def init_helm_command
- command = %w[helm init] + init_command_flags
-
- command.shelljoin
- end
-
- def init_command_flags
- tls_flags + optional_service_account_flag
- end
-
- def tls_flags
- [
- '--tiller-tls',
- '--tiller-tls-verify',
- '--tls-ca-cert', "#{files_dir}/ca.pem",
- '--tiller-tls-cert', "#{files_dir}/cert.pem",
- '--tiller-tls-key', "#{files_dir}/key.pem"
- ]
- end
-
- def optional_service_account_flag
- return [] unless rbac?
-
- ['--service-account', service_account_name]
- end
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/kubernetes/helm/v2/install_command.rb b/lib/gitlab/kubernetes/helm/v2/install_command.rb
deleted file mode 100644
index c50db6bf177..00000000000
--- a/lib/gitlab/kubernetes/helm/v2/install_command.rb
+++ /dev/null
@@ -1,87 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Kubernetes
- module Helm
- module V2
- class InstallCommand < BaseCommand
- include ClientCommand
-
- attr_reader :chart, :repository, :preinstall, :postinstall
- attr_accessor :version
-
- def initialize(chart:, version: nil, repository: nil, preinstall: nil, postinstall: nil, **args)
- super(**args)
- @chart = chart
- @version = version
- @repository = repository
- @preinstall = preinstall
- @postinstall = postinstall
- end
-
- def generate_script
- super + [
- init_command,
- repository_command,
- repository_update_command,
- preinstall,
- install_command,
- postinstall
- ].compact.join("\n")
- end
-
- private
-
- # Uses `helm upgrade --install` which means we can use this for both
- # installation and uprade of applications
- def install_command
- command = ['helm', 'upgrade', name, chart] +
- install_flag +
- rollback_support_flag +
- reset_values_flag +
- optional_version_flag +
- rbac_create_flag +
- namespace_flag +
- value_flag
-
- command.shelljoin
- end
-
- def install_flag
- ['--install']
- end
-
- def reset_values_flag
- ['--reset-values']
- end
-
- def value_flag
- ['-f', "/data/helm/#{name}/config/values.yaml"]
- end
-
- def namespace_flag
- ['--namespace', Gitlab::Kubernetes::Helm::NAMESPACE]
- end
-
- def rbac_create_flag
- if rbac?
- %w[--set rbac.create=true,rbac.enabled=true]
- else
- %w[--set rbac.create=false,rbac.enabled=false]
- end
- end
-
- def optional_version_flag
- return [] unless version
-
- ['--version', version]
- end
-
- def rollback_support_flag
- ['--atomic', '--cleanup-on-fail']
- end
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/kubernetes/helm/v2/patch_command.rb b/lib/gitlab/kubernetes/helm/v2/patch_command.rb
deleted file mode 100644
index 40e56771e47..00000000000
--- a/lib/gitlab/kubernetes/helm/v2/patch_command.rb
+++ /dev/null
@@ -1,67 +0,0 @@
-# frozen_string_literal: true
-
-# PatchCommand is for updating values in installed charts without overwriting
-# existing values.
-module Gitlab
- module Kubernetes
- module Helm
- module V2
- class PatchCommand < BaseCommand
- include ClientCommand
-
- attr_reader :chart, :repository
- attr_accessor :version
-
- def initialize(chart:, version:, repository: nil, **args)
- super(**args)
-
- # version is mandatory to prevent chart mismatches
- # we do not want our values interpreted in the context of the wrong version
- raise ArgumentError, 'version is required' if version.blank?
-
- @chart = chart
- @version = version
- @repository = repository
- end
-
- def generate_script
- super + [
- init_command,
- repository_command,
- repository_update_command,
- upgrade_command
- ].compact.join("\n")
- end
-
- private
-
- def upgrade_command
- command = ['helm', 'upgrade', name, chart] +
- reuse_values_flag +
- version_flag +
- namespace_flag +
- value_flag
-
- command.shelljoin
- end
-
- def reuse_values_flag
- ['--reuse-values']
- end
-
- def value_flag
- ['-f', "/data/helm/#{name}/config/values.yaml"]
- end
-
- def namespace_flag
- ['--namespace', Gitlab::Kubernetes::Helm::NAMESPACE]
- end
-
- def version_flag
- ['--version', version]
- end
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/kubernetes/helm/v2/reset_command.rb b/lib/gitlab/kubernetes/helm/v2/reset_command.rb
deleted file mode 100644
index 00626501a9a..00000000000
--- a/lib/gitlab/kubernetes/helm/v2/reset_command.rb
+++ /dev/null
@@ -1,30 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Kubernetes
- module Helm
- module V2
- class ResetCommand < BaseCommand
- include ClientCommand
-
- def generate_script
- super + [
- init_command,
- reset_helm_command
- ].join("\n")
- end
-
- def pod_name
- "uninstall-#{name}"
- end
-
- private
-
- def reset_helm_command
- 'helm reset --force'
- end
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/kubernetes/helm/v3/base_command.rb b/lib/gitlab/kubernetes/helm/v3/base_command.rb
deleted file mode 100644
index ca1bf5462f0..00000000000
--- a/lib/gitlab/kubernetes/helm/v3/base_command.rb
+++ /dev/null
@@ -1,101 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Kubernetes
- module Helm
- module V3
- class BaseCommand
- attr_reader :name, :files
-
- HELM_VERSION = '3.2.4'
-
- def initialize(rbac:, name:, files:)
- @rbac = rbac
- @name = name
- @files = files
- end
-
- def env
- {}
- end
-
- def rbac?
- @rbac
- end
-
- def pod_resource
- pod_service_account_name = rbac? ? service_account_name : nil
-
- Gitlab::Kubernetes::Helm::Pod.new(self, namespace, service_account_name: pod_service_account_name).generate
- end
-
- def generate_script
- <<~HEREDOC
- set -xeo pipefail
- HEREDOC
- end
-
- def pod_name
- "install-#{name}"
- end
-
- def config_map_resource
- Gitlab::Kubernetes::ConfigMap.new(name, files).generate
- end
-
- def service_account_resource
- return unless rbac?
-
- Gitlab::Kubernetes::ServiceAccount.new(service_account_name, namespace).generate
- end
-
- def cluster_role_binding_resource
- return unless rbac?
-
- subjects = [{ kind: 'ServiceAccount', name: service_account_name, namespace: namespace }]
-
- Gitlab::Kubernetes::ClusterRoleBinding.new(
- cluster_role_binding_name,
- cluster_role_name,
- subjects
- ).generate
- end
-
- def file_names
- files.keys
- end
-
- def repository_command
- ['helm', 'repo', 'add', name, repository].shelljoin if repository
- end
-
- private
-
- def repository_update_command
- 'helm repo update'
- end
-
- def namespace_flag
- ['--namespace', Gitlab::Kubernetes::Helm::NAMESPACE]
- end
-
- def namespace
- Gitlab::Kubernetes::Helm::NAMESPACE
- end
-
- def service_account_name
- Gitlab::Kubernetes::Helm::SERVICE_ACCOUNT
- end
-
- def cluster_role_binding_name
- Gitlab::Kubernetes::Helm::CLUSTER_ROLE_BINDING
- end
-
- def cluster_role_name
- Gitlab::Kubernetes::Helm::CLUSTER_ROLE
- end
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/kubernetes/helm/v3/delete_command.rb b/lib/gitlab/kubernetes/helm/v3/delete_command.rb
deleted file mode 100644
index f628e852f54..00000000000
--- a/lib/gitlab/kubernetes/helm/v3/delete_command.rb
+++ /dev/null
@@ -1,35 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Kubernetes
- module Helm
- module V3
- class DeleteCommand < BaseCommand
- attr_reader :predelete, :postdelete
-
- def initialize(predelete: nil, postdelete: nil, **args)
- super(**args)
- @predelete = predelete
- @postdelete = postdelete
- end
-
- def generate_script
- super + [
- predelete,
- delete_command,
- postdelete
- ].compact.join("\n")
- end
-
- def pod_name
- "uninstall-#{name}"
- end
-
- def delete_command
- ['helm', 'uninstall', name, *namespace_flag].shelljoin
- end
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/kubernetes/helm/v3/install_command.rb b/lib/gitlab/kubernetes/helm/v3/install_command.rb
deleted file mode 100644
index 8d521f0dcd4..00000000000
--- a/lib/gitlab/kubernetes/helm/v3/install_command.rb
+++ /dev/null
@@ -1,80 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Kubernetes
- module Helm
- module V3
- class InstallCommand < BaseCommand
- attr_reader :chart, :repository, :preinstall, :postinstall
- attr_accessor :version
-
- def initialize(chart:, version: nil, repository: nil, preinstall: nil, postinstall: nil, **args)
- super(**args)
- @chart = chart
- @version = version
- @repository = repository
- @preinstall = preinstall
- @postinstall = postinstall
- end
-
- def generate_script
- super + [
- repository_command,
- repository_update_command,
- preinstall,
- install_command,
- postinstall
- ].compact.join("\n")
- end
-
- private
-
- # Uses `helm upgrade --install` which means we can use this for both
- # installation and uprade of applications
- def install_command
- command = ['helm', 'upgrade', name, chart] +
- install_flag +
- rollback_support_flag +
- reset_values_flag +
- optional_version_flag +
- rbac_create_flag +
- namespace_flag +
- value_flag
-
- command.shelljoin
- end
-
- def install_flag
- ['--install']
- end
-
- def reset_values_flag
- ['--reset-values']
- end
-
- def value_flag
- ['-f', "/data/helm/#{name}/config/values.yaml"]
- end
-
- def rbac_create_flag
- if rbac?
- %w[--set rbac.create=true,rbac.enabled=true]
- else
- %w[--set rbac.create=false,rbac.enabled=false]
- end
- end
-
- def optional_version_flag
- return [] unless version
-
- ['--version', version]
- end
-
- def rollback_support_flag
- ['--atomic', '--cleanup-on-fail']
- end
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/kubernetes/helm/v3/patch_command.rb b/lib/gitlab/kubernetes/helm/v3/patch_command.rb
deleted file mode 100644
index 1278e524bd2..00000000000
--- a/lib/gitlab/kubernetes/helm/v3/patch_command.rb
+++ /dev/null
@@ -1,60 +0,0 @@
-# frozen_string_literal: true
-
-# PatchCommand is for updating values in installed charts without overwriting
-# existing values.
-module Gitlab
- module Kubernetes
- module Helm
- module V3
- class PatchCommand < BaseCommand
- attr_reader :chart, :repository
- attr_accessor :version
-
- def initialize(chart:, version:, repository: nil, **args)
- super(**args)
-
- # version is mandatory to prevent chart mismatches
- # we do not want our values interpreted in the context of the wrong version
- raise ArgumentError, 'version is required' if version.blank?
-
- @chart = chart
- @version = version
- @repository = repository
- end
-
- def generate_script
- super + [
- repository_command,
- repository_update_command,
- upgrade_command
- ].compact.join("\n")
- end
-
- private
-
- def upgrade_command
- command = ['helm', 'upgrade', name, chart] +
- reuse_values_flag +
- version_flag +
- namespace_flag +
- value_flag
-
- command.shelljoin
- end
-
- def reuse_values_flag
- ['--reuse-values']
- end
-
- def value_flag
- ['-f', "/data/helm/#{name}/config/values.yaml"]
- end
-
- def version_flag
- ['--version', version]
- end
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/language_detection.rb b/lib/gitlab/language_detection.rb
index b259f58350b..68deafc68b2 100644
--- a/lib/gitlab/language_detection.rb
+++ b/lib/gitlab/language_detection.rb
@@ -45,11 +45,11 @@ module Gitlab
# Returns the ids of the programming languages that do not occur in the detection
# as current repository languages
def deletions
- @repository_languages.map do |repo_lang|
+ @repository_languages.filter_map do |repo_lang|
next if detection.key?(repo_lang.name)
repo_lang.programming_language_id
- end.compact
+ end
end
private
diff --git a/lib/gitlab/legacy_github_import/client.rb b/lib/gitlab/legacy_github_import/client.rb
index dd1502bbbcd..16c3bc09c4d 100644
--- a/lib/gitlab/legacy_github_import/client.rb
+++ b/lib/gitlab/legacy_github_import/client.rb
@@ -34,6 +34,7 @@ module Gitlab
}
)
end
+ alias_method :octokit, :api
def client
unless config
diff --git a/lib/gitlab/legacy_github_import/importer.rb b/lib/gitlab/legacy_github_import/importer.rb
index 331eab7b62a..5eeea8f9548 100644
--- a/lib/gitlab/legacy_github_import/importer.rb
+++ b/lib/gitlab/legacy_github_import/importer.rb
@@ -22,7 +22,7 @@ module Gitlab
unless credentials
raise Projects::ImportService::Error,
- "Unable to find project import data credentials for project ID: #{@project.id}"
+ "Unable to find project import data credentials for project ID: #{@project.id}"
end
opts = {}
@@ -55,9 +55,7 @@ module Gitlab
import_comments(:issues)
# Gitea doesn't have an API endpoint for pull requests comments
- unless project.gitea_import?
- import_comments(:pull_requests)
- end
+ import_comments(:pull_requests) unless project.gitea_import?
import_wiki
@@ -67,9 +65,7 @@ module Gitlab
# See:
# 1) https://gitlab.com/gitlab-org/gitlab/-/issues/343448#note_985979730
# 2) https://gitlab.com/gitlab-org/gitlab/-/merge_requests/89694/diffs#dfc4a8141aa296465ea3c50b095a30292fb6ebc4_180_182
- unless project.gitea_import?
- import_releases
- end
+ import_releases unless project.gitea_import?
handle_errors
@@ -142,8 +138,8 @@ module Gitlab
# rubocop: enable CodeReuse/ActiveRecord
def import_pull_requests
- fetch_resources(:pull_requests, repo, state: :all, sort: :created, direction: :asc, per_page: 100) do |pull_requests|
- pull_requests.each do |raw|
+ fetch_resources(:pull_requests, repo, state: :all, sort: :created, direction: :asc, per_page: 100) do |prs|
+ prs.each do |raw|
raw = raw.to_h
gh_pull_request = PullRequestFormatter.new(project, raw, client)
@@ -156,11 +152,13 @@ module Gitlab
merge_request = gh_pull_request.create!
# Gitea doesn't return PR in the Issue API endpoint, so labels must be assigned at this stage
- if project.gitea_import?
- apply_labels(merge_request, raw)
- end
+ apply_labels(merge_request, raw) if project.gitea_import?
rescue StandardError => e
- errors << { type: :pull_request, url: Gitlab::UrlSanitizer.sanitize(gh_pull_request.url), errors: e.message }
+ errors << {
+ type: :pull_request,
+ url: Gitlab::UrlSanitizer.sanitize(gh_pull_request.url),
+ errors: e.message
+ }
ensure
clean_up_restored_branches(gh_pull_request)
end
@@ -196,9 +194,7 @@ module Gitlab
return unless raw[:labels].count > 0
- label_ids = raw[:labels]
- .map { |attrs| @labels[attrs[:name]] }
- .compact
+ label_ids = raw[:labels].filter_map { |attrs| @labels[attrs[:name]] }
issuable.update_attribute(:label_ids, label_ids)
end
@@ -208,10 +204,14 @@ module Gitlab
resource_type = "#{issuable_type}_comments".to_sym
# Two notes here:
- # 1. We don't have a distinctive attribute for comments (unlike issues iid), so we fetch the last inserted note,
- # compare it against every comment in the current imported page until we find match, and that's where start importing
- # 2. GH returns comments for _both_ issues and PRs through issues_comments API, while pull_requests_comments returns
- # only comments on diffs, so select last note not based on noteable_type but on line_code
+ # 1. We don't have a distinctive attribute for comments (unlike issues
+ # iid), so we fetch the last inserted note, compare it against every
+ # comment in the current imported page until we find match, and that's
+ # where start importing
+ # 2. GH returns comments for _both_ issues and PRs through
+ # issues_comments API, while pull_requests_comments returns only
+ # comments on diffs, so select last note not based on noteable_type but
+ # on line_code
line_code_is = issuable_type == :pull_requests ? 'NOT NULL' : 'NULL'
last_note = project.notes.where("line_code IS #{line_code_is}").last
@@ -264,7 +264,8 @@ module Gitlab
comment_attrs.with_indifferent_access == last_note_attrs
end
- # No matching resource in the collection, which means we got halted right on the end of the last page, so all good
+ # No matching resource in the collection, which means we got halted
+ # right on the end of the last page, so all good
return unless cut_off_index
# Otherwise, remove the resources we've already inserted
@@ -280,9 +281,7 @@ module Gitlab
# GitHub error message when the wiki repo has not been created,
# this means that repo has wiki enabled, but have no pages. So,
# we can skip the import.
- if e.message !~ /repository not exported/
- errors << { type: :wiki, errors: e.message }
- end
+ errors << { type: :wiki, errors: e.message } if e.message.exclude?('repository not exported')
end
def import_releases
diff --git a/lib/gitlab/legacy_github_import/user_formatter.rb b/lib/gitlab/legacy_github_import/user_formatter.rb
index d45a166d2b7..8fd8354e59c 100644
--- a/lib/gitlab/legacy_github_import/user_formatter.rb
+++ b/lib/gitlab/legacy_github_import/user_formatter.rb
@@ -5,6 +5,8 @@ module Gitlab
class UserFormatter
attr_reader :client, :raw
+ GITEA_GHOST_EMAIL = 'ghost_user@gitea_import_dummy_email.com'
+
def initialize(client, raw)
@client = client
@raw = raw
@@ -27,7 +29,14 @@ module Gitlab
private
def email
- @email ||= client.user(raw[:login]).to_h[:email]
+ # Gitea marks deleted users as 'Ghost' users and removes them from
+ # their system. So for Gitea 'Ghost' users we need to assign a dummy
+ # email address to avoid querying the Gitea api for a non existing user
+ if raw[:login] == 'Ghost' && raw[:id] == -1
+ @email = GITEA_GHOST_EMAIL
+ else
+ @email ||= client.user(raw[:login]).to_h[:email]
+ end
end
def find_by_email
diff --git a/lib/gitlab/loggable.rb b/lib/gitlab/loggable.rb
new file mode 100644
index 00000000000..393e3eb37b6
--- /dev/null
+++ b/lib/gitlab/loggable.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Loggable
+ ANONYMOUS = '<Anonymous>'
+
+ def build_structured_payload(**params)
+ { class: self.class.name || ANONYMOUS }.merge(params).stringify_keys
+ end
+ end
+end
diff --git a/lib/gitlab/mail_room.rb b/lib/gitlab/mail_room.rb
index bad2e265f73..5f760e764c8 100644
--- a/lib/gitlab/mail_room.rb
+++ b/lib/gitlab/mail_room.rb
@@ -11,6 +11,20 @@ require_relative 'redis/queues' unless defined?(Gitlab::Redis::Queues)
# This service is run independently of the main Rails process,
# therefore the `Rails` class and its methods are unavailable.
+# TODO: Remove this once we're on Ruby 3
+# https://gitlab.com/gitlab-org/gitlab/-/issues/393651
+unless YAML.respond_to?(:safe_load_file)
+ module YAML
+ # Temporary Ruby 2 back-compat workaround.
+ #
+ # This method only exists as of stdlib 3.0.0:
+ # https://ruby-doc.org/stdlib-3.0.0/libdoc/psych/rdoc/Psych.html
+ def self.safe_load_file(path, **options)
+ YAML.safe_load(File.read(path), **options)
+ end
+ end
+end
+
module Gitlab
module MailRoom
RAILS_ROOT_DIR = Pathname.new('../..').expand_path(__dir__).freeze
@@ -129,7 +143,7 @@ module Gitlab
end
def load_yaml
- @yaml ||= YAML.load_file(config_file)[rails_env].deep_symbolize_keys
+ @yaml ||= YAML.safe_load_file(config_file, aliases: true)[rails_env].deep_symbolize_keys
end
def application_secrets_file
diff --git a/lib/gitlab/metrics/dashboard/finder.rb b/lib/gitlab/metrics/dashboard/finder.rb
index a4964ae0ebc..12f7c347b2d 100644
--- a/lib/gitlab/metrics/dashboard/finder.rb
+++ b/lib/gitlab/metrics/dashboard/finder.rb
@@ -64,7 +64,7 @@ module Gitlab
# display_name: String,
# default: Boolean }]
def find_all_paths(project)
- dashboards = user_facing_dashboard_services(project).flat_map do |service|
+ dashboards = user_facing_dashboard_services.flat_map do |service|
service.all_dashboard_paths(project)
end
@@ -73,19 +73,8 @@ module Gitlab
private
- def user_facing_dashboard_services(project)
- predefined_dashboard_services_for(project) + [project_service]
- end
-
- def predefined_dashboard_services_for(project)
- # Only list the self-monitoring dashboard on the self-monitoring project,
- # since it is the only dashboard (at time of writing) that shows data
- # about GitLab itself.
- if project.self_monitoring?
- return [self_monitoring_service]
- end
-
- PREDEFINED_DASHBOARD_LIST
+ def user_facing_dashboard_services
+ PREDEFINED_DASHBOARD_LIST + [project_service]
end
def system_service
@@ -96,10 +85,6 @@ module Gitlab
::Metrics::Dashboard::CustomDashboardService
end
- def self_monitoring_service
- ::Metrics::Dashboard::SelfMonitoringDashboardService
- end
-
def service_for(options)
Gitlab::Metrics::Dashboard::ServiceSelector.call(options)
end
diff --git a/lib/gitlab/metrics/dashboard/service_selector.rb b/lib/gitlab/metrics/dashboard/service_selector.rb
index 6d4b49676e5..67bf4ce7e9a 100644
--- a/lib/gitlab/metrics/dashboard/service_selector.rb
+++ b/lib/gitlab/metrics/dashboard/service_selector.rb
@@ -23,7 +23,6 @@ module Gitlab
::Metrics::Dashboard::DefaultEmbedService,
::Metrics::Dashboard::SystemDashboardService,
::Metrics::Dashboard::PodDashboardService,
- ::Metrics::Dashboard::SelfMonitoringDashboardService,
::Metrics::Dashboard::CustomDashboardService
].freeze
diff --git a/lib/gitlab/metrics/requests_rack_middleware.rb b/lib/gitlab/metrics/requests_rack_middleware.rb
index f635deabf76..b4baeba72e8 100644
--- a/lib/gitlab/metrics/requests_rack_middleware.rb
+++ b/lib/gitlab/metrics/requests_rack_middleware.rb
@@ -22,7 +22,7 @@ module Gitlab
# reasonable default. If we initialize every category we'll end up
# with an explosion in unused metric combinations, but we want the
# most common ones to be always present.
- FEATURE_CATEGORIES_TO_INITIALIZE = ['authentication_and_authorization',
+ FEATURE_CATEGORIES_TO_INITIALIZE = ['system_access',
'code_review_workflow', 'continuous_integration',
'not_owned', 'source_code_management',
FEATURE_CATEGORY_DEFAULT].freeze
diff --git a/lib/gitlab/metrics/sidekiq_slis.rb b/lib/gitlab/metrics/sidekiq_slis.rb
new file mode 100644
index 00000000000..f28cf4ac967
--- /dev/null
+++ b/lib/gitlab/metrics/sidekiq_slis.rb
@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Metrics
+ module SidekiqSlis
+ EXECUTION_URGENCY_DURATIONS = {
+ "high" => 10,
+ "low" => 300,
+ "throttled" => 300
+ }.freeze
+ # workers without urgency attribute have "low" urgency by default in
+ # WorkerAttributes.get_urgency, just mirroring it here
+ DEFAULT_EXECUTION_URGENCY_DURATION = EXECUTION_URGENCY_DURATIONS["low"]
+
+ class << self
+ def initialize_slis!(possible_labels)
+ Gitlab::Metrics::Sli::Apdex.initialize_sli(:sidekiq_execution, possible_labels)
+ Gitlab::Metrics::Sli::ErrorRate.initialize_sli(:sidekiq_execution, possible_labels)
+ end
+
+ def record_execution_apdex(labels, job_completion_duration)
+ urgency_requirement = execution_duration_for_urgency(labels[:urgency])
+ Gitlab::Metrics::Sli::Apdex[:sidekiq_execution].increment(
+ labels: labels,
+ success: job_completion_duration < urgency_requirement
+ )
+ end
+
+ def record_execution_error(labels, error)
+ Gitlab::Metrics::Sli::ErrorRate[:sidekiq_execution].increment(labels: labels, error: error)
+ end
+
+ def execution_duration_for_urgency(urgency)
+ EXECUTION_URGENCY_DURATIONS.fetch(urgency, DEFAULT_EXECUTION_URGENCY_DURATION)
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/metrics/subscribers/action_cable.rb b/lib/gitlab/metrics/subscribers/action_cable.rb
index 9f955dfe79f..3cd4b6de62b 100644
--- a/lib/gitlab/metrics/subscribers/action_cable.rb
+++ b/lib/gitlab/metrics/subscribers/action_cable.rb
@@ -6,13 +6,18 @@ module Gitlab
class ActionCable < ActiveSupport::Subscriber
include Gitlab::Utils::StrongMemoize
+ SOURCE_DIRECT = 'channel'
+ SOURCE_GRAPHQL_EVENT = 'graphql-event'
+ SOURCE_GRAPHQL_SUBSCRIPTION = 'graphql-subscription'
+ SOURCE_OTHER = 'unknown'
+
attach_to :action_cable
SINGLE_CLIENT_TRANSMISSION = :action_cable_single_client_transmissions_total
TRANSMIT_SUBSCRIPTION_CONFIRMATION = :action_cable_subscription_confirmations_total
TRANSMIT_SUBSCRIPTION_REJECTION = :action_cable_subscription_rejections_total
BROADCAST = :action_cable_broadcasts_total
- DATA_TRANSMITTED_BYTES = :action_cable_transmitted_bytes
+ DATA_TRANSMITTED_BYTES = :action_cable_transmitted_bytes_total
def transmit_subscription_confirmation(event)
confirm_subscription_counter.increment
@@ -23,28 +28,57 @@ module Gitlab
end
def transmit(event)
- transmit_counter.increment
+ payload = event.payload
- if event.payload.present?
- channel = event.payload[:channel_class]
- operation = operation_name_from(event.payload)
- data_size = Gitlab::Json.generate(event.payload[:data]).bytesize
+ labels = {
+ channel: payload[:channel_class],
+ caller: normalize_source(payload[:via])
+ }
+ labels[:broadcasting] = graphql_event_broadcasting_from(payload[:data])
- transmitted_bytes_histogram.observe({ channel: channel, operation: operation }, data_size)
- end
+ transmit_counter.increment(labels)
+ data_size = Gitlab::Json.generate(payload[:data]).bytesize
+ transmitted_bytes_counter.increment(labels, data_size)
end
def broadcast(event)
- broadcast_counter.increment
+ broadcast_counter.increment({ broadcasting: normalize_source(event.payload[:broadcasting]) })
end
private
- # When possible tries to query operation name
- def operation_name_from(payload)
- data = payload.dig(:data, 'result', 'data') || {}
+ # Since transmission sources can have high dimensionality when they carry IDs, we need to
+ # collapse them. If it's not a well-know broadcast, we report it as "other".
+ def normalize_source(source)
+ return SOURCE_DIRECT if source.blank?
+
+ normalized_source = source.gsub('streamed from ', '')
+
+ if normalized_source.start_with?(SOURCE_GRAPHQL_EVENT)
+ # Take at most two levels of topic namespacing.
+ normalized_source.split(':').reject(&:empty?).take(2).join(':') # rubocop: disable CodeReuse/ActiveRecord
+ elsif normalized_source.start_with?(SOURCE_GRAPHQL_SUBSCRIPTION)
+ SOURCE_GRAPHQL_SUBSCRIPTION
+ else
+ SOURCE_OTHER
+ end
+ end
- data.each_key.first
+ # When possible tries to query operation name. This will only return data
+ # for GraphQL subscription broadcasts.
+ def graphql_event_broadcasting_from(payload_data)
+ # Depending on whether the query result was passed in-process from a direct
+ # execution (e.g. in response to a subcription request) or cross-process by
+ # going through PubSub, we might encounter either string or symbol keys.
+ # We do not use deep_transform_keys here because the payload can be large
+ # and performance would be affected.
+ query_result = payload_data[:result] || payload_data['result'] || {}
+ query_result_data = query_result['data'] || {}
+ gql_operation = query_result_data.each_key.first
+
+ return unless gql_operation
+
+ "#{SOURCE_GRAPHQL_EVENT}:#{gql_operation}"
end
def transmit_counter
@@ -83,9 +117,12 @@ module Gitlab
end
end
- def transmitted_bytes_histogram
- strong_memoize("transmitted_bytes_histogram") do
- ::Gitlab::Metrics.histogram(DATA_TRANSMITTED_BYTES, 'Message size, in bytes, transmitted over action cable')
+ def transmitted_bytes_counter
+ strong_memoize("transmitted_bytes_counter") do
+ ::Gitlab::Metrics.counter(
+ DATA_TRANSMITTED_BYTES,
+ 'Total number of bytes transmitted over ActionCable'
+ )
end
end
end
diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb
index e3756a8c9f6..10bb358a292 100644
--- a/lib/gitlab/metrics/subscribers/active_record.rb
+++ b/lib/gitlab/metrics/subscribers/active_record.rb
@@ -9,7 +9,6 @@ module Gitlab
attach_to :active_record
- IGNORABLE_SQL = %w{BEGIN COMMIT}.freeze
DB_COUNTERS = %i{count write_count cached_count}.freeze
SQL_COMMANDS_WITH_COMMENTS_REGEX = %r{\A(/\*.*\*/\s)?((?!(.*[^\w'"](DELETE|UPDATE|INSERT INTO)[^\w'"])))(WITH.*)?(SELECT)((?!(FOR UPDATE|FOR SHARE)).)*$}i.freeze
@@ -114,7 +113,7 @@ module Gitlab
end
def ignored_query?(payload)
- payload[:name] == 'SCHEMA' || IGNORABLE_SQL.include?(payload[:sql])
+ payload[:name] == 'SCHEMA' || payload[:name] == 'TRANSACTION'
end
def cached_query?(payload)
diff --git a/lib/gitlab/metrics/subscribers/external_http.rb b/lib/gitlab/metrics/subscribers/external_http.rb
index ff8654a2cec..87756b14887 100644
--- a/lib/gitlab/metrics/subscribers/external_http.rb
+++ b/lib/gitlab/metrics/subscribers/external_http.rb
@@ -13,6 +13,10 @@ module Gitlab
DETAIL_STORE = :external_http_detail_store
COUNTER = :external_http_count
DURATION = :external_http_duration_s
+ SLOW_REQUESTS = :external_http_slow_requests
+
+ THRESHOLD_SLOW_REQUEST_S = 5.0
+ MAX_SLOW_REQUESTS = 10
def self.detail_store
::Gitlab::SafeRequestStore[DETAIL_STORE] ||= []
@@ -26,11 +30,24 @@ module Gitlab
Gitlab::SafeRequestStore[COUNTER].to_i
end
+ def self.slow_requests
+ Gitlab::SafeRequestStore[SLOW_REQUESTS]
+ end
+
+ def self.top_slowest_requests
+ requests = slow_requests
+
+ return unless requests.present?
+
+ requests.sort_by { |req| req[:duration_s] }.reverse.first(MAX_SLOW_REQUESTS)
+ end
+
def self.payload
{
COUNTER => request_count,
- DURATION => duration
- }
+ DURATION => duration,
+ SLOW_REQUESTS => top_slowest_requests
+ }.compact
end
def request(event)
@@ -69,6 +86,17 @@ module Gitlab
Gitlab::SafeRequestStore[COUNTER] = Gitlab::SafeRequestStore[COUNTER].to_i + 1
Gitlab::SafeRequestStore[DURATION] = Gitlab::SafeRequestStore[DURATION].to_f + payload[:duration].to_f
+
+ if payload[:duration].to_f > THRESHOLD_SLOW_REQUEST_S
+ Gitlab::SafeRequestStore[SLOW_REQUESTS] ||= []
+ Gitlab::SafeRequestStore[SLOW_REQUESTS] << {
+ method: payload[:method],
+ host: payload[:host],
+ port: payload[:port],
+ path: payload[:path],
+ duration_s: payload[:duration].to_f.round(3)
+ }
+ end
end
def expose_metrics(payload)
diff --git a/lib/gitlab/metrics/subscribers/rack_attack.rb b/lib/gitlab/metrics/subscribers/rack_attack.rb
index e6cf14a6c8c..2196122df01 100644
--- a/lib/gitlab/metrics/subscribers/rack_attack.rb
+++ b/lib/gitlab/metrics/subscribers/rack_attack.rb
@@ -22,11 +22,6 @@ module Gitlab
}
end
- def redis(event)
- self.class.payload[:rack_attack_redis_count] += 1
- self.class.payload[:rack_attack_redis_duration_s] += event.duration.to_f / 1000
- end
-
def safelist(event)
req = event.payload[:request]
Gitlab::Instrumentation::Throttle.safelist = req.env['rack.attack.matched']
diff --git a/lib/gitlab/metrics/subscribers/rails_cache.rb b/lib/gitlab/metrics/subscribers/rails_cache.rb
index b12db9df66d..b4e9e85a012 100644
--- a/lib/gitlab/metrics/subscribers/rails_cache.rb
+++ b/lib/gitlab/metrics/subscribers/rails_cache.rb
@@ -9,18 +9,19 @@ module Gitlab
attach_to :active_support
def cache_read_multi(event)
- observe(:read_multi, event.duration)
+ observe(:read_multi, event)
return unless current_transaction
- current_transaction.observe(:gitlab_cache_read_multikey_count, event.payload[:key].size) do
+ labels = { store: event.payload[:store].split('::').last }
+ current_transaction.observe(:gitlab_cache_read_multikey_count, event.payload[:key].size, labels) do
buckets [10, 50, 100, 1000]
docstring 'Number of keys for mget in read_multi/fetch_multi'
end
end
def cache_read(event)
- observe(:read, event.duration)
+ observe(:read, event)
return unless current_transaction
return if event.payload[:super_operation] == :fetch
@@ -33,15 +34,15 @@ module Gitlab
end
def cache_write(event)
- observe(:write, event.duration)
+ observe(:write, event)
end
def cache_delete(event)
- observe(:delete, event.duration)
+ observe(:delete, event)
end
def cache_exist?(event)
- observe(:exists, event.duration)
+ observe(:exists, event)
end
def cache_fetch_hit(event)
@@ -60,17 +61,17 @@ module Gitlab
current_transaction.increment(:gitlab_transaction_cache_read_miss_count_total, 1)
end
- def observe(key, duration)
+ def observe(key, event)
return unless current_transaction
- labels = { operation: key }
+ labels = { operation: key, store: event.payload[:store].split('::').last }
current_transaction.increment(:gitlab_cache_operations_total, 1, labels) do
docstring 'Cache operations'
label_keys labels.keys
end
- metric_cache_operation_duration_seconds.observe(labels, duration / 1000.0)
+ metric_cache_operation_duration_seconds.observe(labels, event.duration / 1000.0)
end
private
@@ -84,7 +85,7 @@ module Gitlab
:gitlab_cache_operation_duration_seconds,
'Cache access time',
{},
- [0.00001, 0.0001, 0.001, 0.01, 0.1, 1.0]
+ Gitlab::Instrumentation::Redis::QUERY_TIME_BUCKETS
)
end
end
diff --git a/lib/gitlab/middleware/compressed_json.rb b/lib/gitlab/middleware/compressed_json.rb
index 80916eab5ac..cc485d8a5db 100644
--- a/lib/gitlab/middleware/compressed_json.rb
+++ b/lib/gitlab/middleware/compressed_json.rb
@@ -4,15 +4,23 @@ module Gitlab
module Middleware
class CompressedJson
COLLECTOR_PATH = '/api/v4/error_tracking/collector'
- PACKAGES_PATH = %r{
- \A/api/v4/ (?# prefix)
- (?:projects/
- (?<project_id>
- .+ (?# at least one character)
- )/
- )? (?# projects segment)
- packages/npm/-/npm/v1/security/
- (?:(?:advisories/bulk)|(?:audits/quick))\z (?# end)
+ INSTANCE_PACKAGES_PATH = %r{
+ \A/api/v4/packages/npm/-/npm/v1/security/
+ (?:(?:advisories/bulk)|(?:audits/quick))\z (?# end)
+ }xi.freeze
+ GROUP_PACKAGES_PATH = %r{
+ \A/api/v4/groups/
+ (?<id>
+ [a-zA-Z0-9%-._]{1,255}
+ )/-/packages/npm/-/npm/v1/security/
+ (?:(?:advisories/bulk)|(?:audits/quick))\z (?# end)
+ }xi.freeze
+ PROJECT_PACKAGES_PATH = %r{
+ \A/api/v4/projects/
+ (?<id>
+ [a-zA-Z0-9%-._]{1,255}
+ )/packages/npm/-/npm/v1/security/
+ (?:(?:advisories/bulk)|(?:audits/quick))\z (?# end)
}xi.freeze
MAXIMUM_BODY_SIZE = 200.kilobytes.to_i
UNSAFE_CHARACTERS = %r{[!"#&'()*+,./:;<>=?@\[\]^`{}|~$]}xi.freeze
@@ -76,16 +84,19 @@ module Gitlab
end
def match_packages_path?(env)
- match_data = env['PATH_INFO'].delete_prefix(relative_url).match(PACKAGES_PATH)
+ path = env['PATH_INFO'].delete_prefix(relative_url)
+ match_data = path.match(INSTANCE_PACKAGES_PATH) ||
+ path.match(PROJECT_PACKAGES_PATH) ||
+ path.match(GROUP_PACKAGES_PATH)
return false unless match_data
- return true unless match_data[:project_id] # instance level endpoint was matched
+ return true if match_data.names.empty? # instance level endpoint was matched
- url_encoded?(match_data[:project_id])
+ url_encoded?(match_data[:id])
end
- def url_encoded?(project_id)
- project_id !~ UNSAFE_CHARACTERS
+ def url_encoded?(id)
+ id !~ UNSAFE_CHARACTERS
end
end
end
diff --git a/lib/gitlab/middleware/go.rb b/lib/gitlab/middleware/go.rb
index 13f7ab36823..4da5fef9fd7 100644
--- a/lib/gitlab/middleware/go.rb
+++ b/lib/gitlab/middleware/go.rb
@@ -18,7 +18,7 @@ module Gitlab
request = ActionDispatch::Request.new(env)
render_go_doc(request) || @app.call(env)
- rescue Gitlab::Auth::IpBlacklisted
+ rescue Gitlab::Auth::IpBlocked
Gitlab::AuthLogger.error(
message: 'Rack_Attack',
status: 403,
diff --git a/lib/gitlab/middleware/request_context.rb b/lib/gitlab/middleware/request_context.rb
index 07f6f87a68c..f609002007c 100644
--- a/lib/gitlab/middleware/request_context.rb
+++ b/lib/gitlab/middleware/request_context.rb
@@ -8,15 +8,9 @@ module Gitlab
end
def call(env)
- # We should be using ActionDispatch::Request instead of
- # Rack::Request to be consistent with Rails, but due to a Rails
- # bug described in
- # https://gitlab.com/gitlab-org/gitlab-foss/issues/58573#note_149799010
- # hosts behind a load balancer will only see 127.0.0.1 for the
- # load balancer's IP.
- req = Rack::Request.new(env)
-
- ::Gitlab::InstrumentationHelper.init_instrumentation_data(request_ip: req.ip)
+ request = ActionDispatch::Request.new(env)
+ Gitlab::RequestContext.start_request_context(request: request)
+ Gitlab::RequestContext.start_thread_context
@app.call(env)
end
diff --git a/lib/gitlab/multi_collection_paginator.rb b/lib/gitlab/multi_collection_paginator.rb
index 87cc0a0d3d2..e122f0b9317 100644
--- a/lib/gitlab/multi_collection_paginator.rb
+++ b/lib/gitlab/multi_collection_paginator.rb
@@ -9,6 +9,8 @@ module Gitlab
@per_page = (per_page || Kaminari.config.default_per_page).to_i
@first_collection, @second_collection = collections
+
+ raise ArgumentError, 'Page size must be at least 1' if @per_page < 1
end
def paginate(page)
diff --git a/lib/gitlab/nav/top_nav_menu_item.rb b/lib/gitlab/nav/top_nav_menu_item.rb
index 75eb0e8a264..a83cdbe15df 100644
--- a/lib/gitlab/nav/top_nav_menu_item.rb
+++ b/lib/gitlab/nav/top_nav_menu_item.rb
@@ -8,7 +8,10 @@ module Gitlab
# this is already :/. We could also take a hash and manually check every
# entry, but it's much more maintainable to do rely on native Ruby.
# rubocop: disable Metrics/ParameterLists
- def self.build(id:, title:, active: false, icon: '', href: '', view: '', css_class: nil, data: nil, emoji: nil)
+ def self.build(
+ id:, title:, active: false, icon: '', href: '', view: '',
+ css_class: nil, data: nil, partial: nil, component: nil
+ )
{
id: id,
type: :item,
@@ -19,7 +22,8 @@ module Gitlab
view: view.to_s,
css_class: css_class,
data: data || { qa_selector: 'menu_item_link', qa_title: title },
- emoji: emoji
+ partial: partial,
+ component: component
}
end
# rubocop: enable Metrics/ParameterLists
diff --git a/lib/gitlab/no_cache_headers.rb b/lib/gitlab/no_cache_headers.rb
index 2d03741480d..6afb108dcfd 100644
--- a/lib/gitlab/no_cache_headers.rb
+++ b/lib/gitlab/no_cache_headers.rb
@@ -4,7 +4,6 @@ module Gitlab
module NoCacheHeaders
DEFAULT_GITLAB_NO_CACHE_HEADERS = {
'Cache-Control' => "#{ActionDispatch::Http::Cache::Response::DEFAULT_CACHE_CONTROL}, no-store, no-cache",
- 'Pragma' => 'no-cache', # HTTP 1.0 compatibility
'Expires' => 'Fri, 01 Jan 1990 00:00:00 GMT'
}.freeze
diff --git a/lib/gitlab/observability.rb b/lib/gitlab/observability.rb
index 8dbd2f41ccb..f7f65c91339 100644
--- a/lib/gitlab/observability.rb
+++ b/lib/gitlab/observability.rb
@@ -2,8 +2,19 @@
module Gitlab
module Observability
- module_function
+ extend self
+ ACTION_TO_PERMISSION = {
+ explore: :read_observability,
+ datasources: :admin_observability,
+ manage: :admin_observability,
+ dashboards: :read_observability
+ }.freeze
+
+ EMBEDDABLE_PATHS = %w[explore goto].freeze
+
+ # Returns the GitLab Observability URL
+ #
def observability_url
return ENV['OVERRIDE_OBSERVABILITY_URL'] if ENV['OVERRIDE_OBSERVABILITY_URL']
# TODO Make observability URL configurable https://gitlab.com/gitlab-org/opstrace/opstrace-ui/-/issues/80
@@ -12,8 +23,123 @@ module Gitlab
'https://observe.gitlab.com'
end
- def observability_enabled?(user, group)
- Gitlab::Observability.observability_url.present? && Ability.allowed?(user, :read_observability, group)
+ # Returns true if the Observability feature flag is enabled
+ #
+ def enabled?(group = nil)
+ return Feature.enabled?(:observability_group_tab, group) if group
+
+ Feature.enabled?(:observability_group_tab)
+ end
+
+ # Returns the embeddable Observability URL of a given URL
+ #
+ # - Validates the URL
+ # - Checks that the path is embeddable
+ # - Converts the gitlab.com URL to observe.gitlab.com URL
+ #
+ # e.g.
+ #
+ # from: gitlab.com/groups/GROUP_PATH/-/observability/explore?observability_path=/explore
+ # to observe.gitlab.com/-/GROUP_ID/explore
+ #
+ # Returns nil if the URL is not a valid Observability URL or the path is not embeddable
+ #
+ def embeddable_url(url)
+ uri = validate_url(url, Gitlab.config.gitlab.url)
+ return unless uri
+
+ group = group_from_observability_url(url)
+ return unless group
+
+ parsed_query = CGI.parse(uri.query.to_s).transform_values(&:first).symbolize_keys
+ observability_path = parsed_query[:observability_path]
+
+ return build_full_url(group, observability_path, '/') if observability_path_embeddable?(observability_path)
+ end
+
+ # Returns true if the user is allowed to perform an action within a group
+ #
+ def allowed_for_action?(user, group, action)
+ return false if action.nil?
+
+ permission = ACTION_TO_PERMISSION.fetch(action.to_sym, :admin_observability)
+ allowed?(user, group, permission)
+ end
+
+ # Returns true if the user has the specified permission within the group
+ def allowed?(user, group, permission = :admin_observability)
+ return false unless group && user
+
+ observability_url.present? && Ability.allowed?(user, permission, group)
+ end
+
+ # Builds the full Observability URL given a certan group and path
+ #
+ # If unsanitized_observability_path is not valid or missing, fallbacks to fallback_path
+ #
+ def build_full_url(group, unsanitized_observability_path, fallback_path)
+ return unless group
+
+ # When running Observability UI in standalone mode (i.e. not backed by Observability Backend)
+ # the group-id is not required. !!This is only used for local dev!!
+ base_url = ENV['STANDALONE_OBSERVABILITY_UI'] == 'true' ? observability_url : "#{observability_url}/-/#{group.id}"
+
+ sanitized_path = if unsanitized_observability_path && sanitize(unsanitized_observability_path) != ''
+ CGI.unescapeHTML(sanitize(unsanitized_observability_path))
+ else
+ fallback_path || '/'
+ end
+
+ sanitized_path.prepend('/') if sanitized_path[0] != '/'
+
+ "#{base_url}#{sanitized_path}"
+ end
+
+ private
+
+ def validate_url(url, reference_url)
+ uri = URI.parse(url)
+ reference_uri = URI.parse(reference_url)
+
+ return uri if uri.scheme == reference_uri.scheme &&
+ uri.port == reference_uri.port &&
+ uri.host.casecmp?(reference_uri.host)
+ rescue URI::InvalidURIError
+ nil
+ end
+
+ def link_sanitizer
+ @link_sanitizer ||= Rails::Html::Sanitizer.safe_list_sanitizer.new
+ end
+
+ def sanitize(input)
+ link_sanitizer.sanitize(input, {})&.html_safe
+ end
+
+ def group_from_observability_url(url)
+ match = Rails.application.routes.recognize_path(url)
+
+ return if match[:unmatched_route].present?
+ return if match[:group_id].blank? || match[:action].blank? || match[:controller] != "groups/observability"
+
+ group_path = match[:group_id]
+ Group.find_by_full_path(group_path)
+ rescue ActionController::RoutingError
+ nil
+ end
+
+ def observability_path_embeddable?(observability_path)
+ return false unless observability_path
+
+ observability_path = observability_path[1..] if observability_path[0] == '/'
+
+ parsed_observability_path = URI.parse(observability_path).path.split('/')
+
+ base_path = parsed_observability_path[0]
+
+ EMBEDDABLE_PATHS.include?(base_path)
+ rescue URI::InvalidURIError
+ false
end
end
end
diff --git a/lib/gitlab/octokit/middleware.rb b/lib/gitlab/octokit/middleware.rb
index a92860f7eb8..f944f9827a3 100644
--- a/lib/gitlab/octokit/middleware.rb
+++ b/lib/gitlab/octokit/middleware.rb
@@ -11,7 +11,8 @@ module Gitlab
Gitlab::UrlBlocker.validate!(env[:url],
schemes: %w[http https],
allow_localhost: allow_local_requests?,
- allow_local_network: allow_local_requests?
+ allow_local_network: allow_local_requests?,
+ dns_rebind_protection: dns_rebind_protection?
)
@app.call(env)
@@ -19,6 +20,10 @@ module Gitlab
private
+ def dns_rebind_protection?
+ Gitlab::CurrentSettings.dns_rebinding_protection_enabled?
+ end
+
def allow_local_requests?
Gitlab::CurrentSettings.allow_local_requests_from_web_hooks_and_services?
end
diff --git a/lib/gitlab/omniauth_initializer.rb b/lib/gitlab/omniauth_initializer.rb
index a03533dcd9a..81ad7a7f9e1 100644
--- a/lib/gitlab/omniauth_initializer.rb
+++ b/lib/gitlab/omniauth_initializer.rb
@@ -21,8 +21,6 @@ module Gitlab
class << self
def default_arguments_for(provider_name)
case provider_name
- when 'cas3'
- { on_single_sign_out: cas3_signout_handler }
when 'shibboleth'
{ fail_with_empty_uid: true }
when 'google_oauth2'
@@ -39,18 +37,6 @@ module Gitlab
def full_host
proc { |_env| Settings.gitlab['base_url'] }
end
-
- private
-
- def cas3_signout_handler
- lambda do |request|
- ticket = request.params[:session_index]
- raise "Service Ticket not found." unless Gitlab::Auth::OAuth::Session.valid?(:cas3, ticket)
-
- Gitlab::Auth::OAuth::Session.destroy(:cas3, ticket)
- true
- end
- end
end
private
@@ -74,7 +60,7 @@ module Gitlab
# An Array from the configuration will be expanded
provider_arguments.concat arguments
provider_arguments << defaults unless defaults.empty?
- when Hash
+ when Hash, GitlabSettings::Options
hash_arguments = arguments.deep_symbolize_keys.deep_merge(defaults)
normalized = normalize_hash_arguments(hash_arguments)
diff --git a/lib/gitlab/optimistic_locking.rb b/lib/gitlab/optimistic_locking.rb
index 9f39b5f122f..3c8ac55f70b 100644
--- a/lib/gitlab/optimistic_locking.rb
+++ b/lib/gitlab/optimistic_locking.rb
@@ -10,8 +10,11 @@ module Gitlab
start_time = Gitlab::Metrics::System.monotonic_time
retry_attempts = 0
+ # prevent scope override, see https://gitlab.com/gitlab-org/gitlab/-/issues/391186
+ klass = subject.is_a?(ActiveRecord::Relation) ? subject.klass : subject.class
+
begin
- subject.transaction do
+ klass.transaction do
yield(subject)
end
rescue ActiveRecord::StaleObjectError
diff --git a/lib/gitlab/other_markup.rb b/lib/gitlab/other_markup.rb
index 2368ea3ad28..5b6be88a46f 100644
--- a/lib/gitlab/other_markup.rb
+++ b/lib/gitlab/other_markup.rb
@@ -22,15 +22,10 @@ module Gitlab
Gitlab::RenderTimeout.timeout(foreground: RENDER_TIMEOUT) { GitHub::Markup.render(file_name, input) }
rescue Timeout::Error => e
class_name = name.demodulize
- timeout_counter.increment(source: class_name)
Gitlab::ErrorTracking.track_exception(e, project_id: context[:project]&.id, class_name: class_name,
file_name: file_name)
ActionController::Base.helpers.simple_format(input)
end
-
- def self.timeout_counter
- Gitlab::Metrics.counter(:banzai_filter_timeouts_total, 'Count of the Banzai filters that time out')
- end
end
end
diff --git a/lib/gitlab/pages/deployment_update.rb b/lib/gitlab/pages/deployment_update.rb
index 2f5c6938e2a..6845f5d88ec 100644
--- a/lib/gitlab/pages/deployment_update.rb
+++ b/lib/gitlab/pages/deployment_update.rb
@@ -46,9 +46,13 @@ module Gitlab
end
end
+ def root_dir
+ build.options[:publish] || PUBLIC_DIR
+ end
+
# Calculate page size after extract
def total_size
- @total_size ||= build.artifacts_metadata_entry(PUBLIC_DIR + '/', recursive: true).total_size
+ @total_size ||= build.artifacts_metadata_entry("#{root_dir}/", recursive: true).total_size
end
def max_size_from_settings
@@ -74,7 +78,10 @@ module Gitlab
def validate_public_folder
if total_size <= 0
- errors.add(:base, 'Error: The `public/` folder is missing, or not declared in `.gitlab-ci.yml`.')
+ errors.add(
+ :base,
+ 'Error: You need to either include a `public/` folder in your artifacts, or specify ' \
+ 'which one to use for Pages using `publish` in `.gitlab-ci.yml`')
end
end
diff --git a/lib/gitlab/pages/random_domain.rb b/lib/gitlab/pages/random_domain.rb
new file mode 100644
index 00000000000..8aa7611c910
--- /dev/null
+++ b/lib/gitlab/pages/random_domain.rb
@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Pages
+ class RandomDomain
+ PROJECT_PATH_LIMIT = 48
+ SUBDOMAIN_LABEL_LIMIT = 63
+
+ def self.generate(project_path:, namespace_path:)
+ new(project_path: project_path, namespace_path: namespace_path).generate
+ end
+
+ def initialize(project_path:, namespace_path:)
+ @project_path = project_path
+ @namespace_path = namespace_path
+ end
+
+ # Subdomains have a limit of 63 bytes (https://www.freesoft.org/CIE/RFC/1035/9.htm)
+ # For this reason we're limiting each part of the unique subdomain
+ #
+ # The domain is made up of 3 parts, like: projectpath-namespacepath-randomstring
+ # - project path: between 1 and 48 chars
+ # - namespace path: when the project path has less than 48 chars,
+ # the namespace full path will be used to fill the value up to 48 chars
+ # - random hexadecimal: to ensure a random value, the domain is then filled
+ # with a random hexadecimal value to complete 63 chars
+ def generate
+ domain = project_path.byteslice(0, PROJECT_PATH_LIMIT)
+
+ # if the project_path has less than PROJECT_PATH_LIMIT chars,
+ # fill the domain with the parent full_path up to 48 chars like:
+ # projectpath-namespacepath
+ if domain.length < PROJECT_PATH_LIMIT
+ namespace_size = PROJECT_PATH_LIMIT - domain.length - 1
+ domain.concat('-', namespace_path.byteslice(0, namespace_size))
+ end
+
+ # Complete the domain with random hexadecimal values util it is 63 chars long
+ # PS.: SecureRandom.hex return an string twice the size passed as argument.
+ domain.concat('-', SecureRandom.hex(SUBDOMAIN_LABEL_LIMIT - domain.length - 1))
+
+ # Slugify ensures the format and size (63 chars) of the given string
+ Gitlab::Utils.slugify(domain)
+ end
+
+ private
+
+ attr_reader :project_path, :namespace_path
+ end
+ end
+end
diff --git a/lib/gitlab/pages/virtual_host_finder.rb b/lib/gitlab/pages/virtual_host_finder.rb
new file mode 100644
index 00000000000..5fec60188f8
--- /dev/null
+++ b/lib/gitlab/pages/virtual_host_finder.rb
@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Pages
+ class VirtualHostFinder
+ def initialize(host)
+ @host = host&.downcase
+ end
+
+ def execute
+ return if host.blank?
+
+ gitlab_host = ::Settings.pages.host.downcase.prepend(".")
+
+ if host.ends_with?(gitlab_host)
+ name = host.delete_suffix(gitlab_host)
+
+ by_namespace_domain(name) ||
+ by_unique_domain(name)
+ else
+ by_custom_domain(host)
+ end
+ end
+
+ private
+
+ attr_reader :host
+
+ def by_unique_domain(name)
+ project = Project.by_pages_enabled_unique_domain(name)
+
+ return unless Feature.enabled?(:pages_unique_domain, project)
+ return unless project&.pages_deployed?
+
+ ::Pages::VirtualDomain.new(projects: [project])
+ end
+
+ def by_namespace_domain(name)
+ namespace = Namespace.top_most.by_path(name)
+
+ return if namespace.blank?
+
+ cache = if Feature.enabled?(:cache_pages_domain_api, namespace)
+ ::Gitlab::Pages::CacheControl.for_namespace(namespace.id)
+ end
+
+ ::Pages::VirtualDomain.new(
+ trim_prefix: namespace.full_path,
+ projects: namespace.all_projects_with_pages,
+ cache: cache
+ )
+ end
+
+ def by_custom_domain(host)
+ domain = PagesDomain.find_by_domain_case_insensitive(host)
+
+ return unless domain&.pages_deployed?
+
+ cache = if Feature.enabled?(:cache_pages_domain_api, domain.project.root_namespace)
+ ::Gitlab::Pages::CacheControl.for_domain(domain.id)
+ end
+
+ ::Pages::VirtualDomain.new(
+ projects: [domain.project],
+ domain: domain,
+ cache: cache
+ )
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/pagination/keyset.rb b/lib/gitlab/pagination/keyset.rb
index 67a5530d46c..56017ba846c 100644
--- a/lib/gitlab/pagination/keyset.rb
+++ b/lib/gitlab/pagination/keyset.rb
@@ -3,12 +3,12 @@
module Gitlab
module Pagination
module Keyset
- SUPPORTED_TYPES = [
+ SUPPORTED_TYPES = %w[
Project
].freeze
def self.available_for_type?(relation)
- SUPPORTED_TYPES.include?(relation.klass)
+ SUPPORTED_TYPES.include?(relation.klass.to_s)
end
def self.available?(request_context, relation)
diff --git a/lib/gitlab/pagination/keyset/paginator.rb b/lib/gitlab/pagination/keyset/paginator.rb
index 1ff4589d8e1..fb670a3f9ea 100644
--- a/lib/gitlab/pagination/keyset/paginator.rb
+++ b/lib/gitlab/pagination/keyset/paginator.rb
@@ -94,8 +94,6 @@ module Gitlab
data = order.cursor_attributes_for_node(records.last)
data[direction_key] = FORWARD_DIRECTION
cursor_converter.dump(data)
- else
- nil
end
end
diff --git a/lib/gitlab/patch/database_config.rb b/lib/gitlab/patch/database_config.rb
index 20d8f7be8fd..8a7566f6e0e 100644
--- a/lib/gitlab/patch/database_config.rb
+++ b/lib/gitlab/patch/database_config.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
# The purpose of this code is to set the migrations path
-# for the Geo tracking database.
+# for the Geo tracking database and the embedding database.
module Gitlab
module Patch
module DatabaseConfig
@@ -10,13 +10,17 @@ module Gitlab
def database_configuration
super.to_h do |env, configs|
if Gitlab.ee?
- if configs.key?("geo")
- migrations_paths = Array(configs["geo"]["migrations_paths"])
- migrations_paths << "ee/db/geo/migrate" if migrations_paths.empty?
- migrations_paths << "ee/db/geo/post_migrate" unless ENV['SKIP_POST_DEPLOYMENT_MIGRATIONS']
+ ee_databases = %w[embedding geo]
- configs["geo"]["migrations_paths"] = migrations_paths.uniq
- configs["geo"]["schema_migrations_path"] = "ee/db/geo/schema_migrations" if configs["geo"]["schema_migrations_path"].blank?
+ ee_databases.each do |ee_db_name|
+ next unless configs.key?(ee_db_name)
+
+ migrations_paths = Array(configs[ee_db_name]['migrations_paths'])
+ migrations_paths << File.join('ee', 'db', ee_db_name, 'migrate') if migrations_paths.empty?
+ migrations_paths << File.join('ee', 'db', ee_db_name, 'post_migrate') unless ENV['SKIP_POST_DEPLOYMENT_MIGRATIONS']
+
+ configs[ee_db_name]['migrations_paths'] = migrations_paths.uniq
+ configs[ee_db_name]['schema_migrations_path'] = File.join('ee', 'db', ee_db_name, 'schema_migrations') if configs[ee_db_name]['schema_migrations_path'].blank?
end
end
diff --git a/lib/gitlab/patch/draw_route.rb b/lib/gitlab/patch/draw_route.rb
index 61b25065e8f..67da6c9c943 100644
--- a/lib/gitlab/patch/draw_route.rb
+++ b/lib/gitlab/patch/draw_route.rb
@@ -27,7 +27,7 @@ module Gitlab
def draw_route(path)
if File.exist?(path)
- instance_eval(File.read(path))
+ instance_eval(File.read(path), path.to_s)
true
else
false
diff --git a/lib/gitlab/patch/node_loader.rb b/lib/gitlab/patch/node_loader.rb
new file mode 100644
index 00000000000..79f4b17dd93
--- /dev/null
+++ b/lib/gitlab/patch/node_loader.rb
@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+# Patch to address https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/2212#note_1287996694
+# It uses hostname instead of IP address if the former is present in `CLUSTER NODES` output.
+if Gem::Version.new(Redis::VERSION) > Gem::Version.new('4.8.1')
+ raise 'New version of redis detected, please remove or update this patch'
+end
+
+module Gitlab
+ module Patch
+ module NodeLoader
+ def self.prepended(base)
+ base.class_eval do
+ # monkey-patches https://github.com/redis/redis-rb/blob/v4.8.0/lib/redis/cluster/node_loader.rb#L23
+ def self.fetch_node_info(node)
+ node.call(%i[cluster nodes]).split("\n").map(&:split).to_h do |arr|
+ [
+ extract_host_identifier(arr[1]),
+ (arr[2].split(',') & %w[master slave]).first # rubocop:disable Naming/InclusiveLanguage
+ ]
+ end
+ end
+
+ # Since `CLUSTER SLOT` uses the preferred endpoint determined by
+ # the `cluster-preferred-endpoint-type` config value, we will prefer hostname over IP address.
+ # See https://redis.io/commands/cluster-nodes/ for details on the output format.
+ #
+ # @param [String] Address info matching fhe format: <ip:port@cport[,hostname[,auxiliary_field=value]*]>
+ def self.extract_host_identifier(node_address)
+ ip_chunk, hostname, _auxiliaries = node_address.split(',')
+ return ip_chunk.split('@').first if hostname.blank?
+
+ port = ip_chunk.split('@').first.split(':')[1]
+ "#{hostname}:#{port}"
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/phabricator_import.rb b/lib/gitlab/phabricator_import.rb
deleted file mode 100644
index 49e01eceb5b..00000000000
--- a/lib/gitlab/phabricator_import.rb
+++ /dev/null
@@ -1,11 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module PhabricatorImport
- BaseError = Class.new(StandardError)
-
- def self.available?
- Gitlab::CurrentSettings.import_sources.include?('phabricator')
- end
- end
-end
diff --git a/lib/gitlab/phabricator_import/cache/map.rb b/lib/gitlab/phabricator_import/cache/map.rb
deleted file mode 100644
index 7aba3cf26fd..00000000000
--- a/lib/gitlab/phabricator_import/cache/map.rb
+++ /dev/null
@@ -1,71 +0,0 @@
-# frozen_string_literal: true
-module Gitlab
- module PhabricatorImport
- module Cache
- class Map
- def initialize(project)
- @project = project
- end
-
- def get_gitlab_model(phabricator_id)
- cached_info = get(phabricator_id)
-
- if cached_info[:classname] && cached_info[:database_id]
- object = cached_info[:classname].constantize.find_by_id(cached_info[:database_id])
- else
- object = yield if block_given?
- set_gitlab_model(object, phabricator_id) if object
- end
-
- object
- end
-
- def set_gitlab_model(object, phabricator_id)
- set(object.class, object.id, phabricator_id)
- end
-
- private
-
- attr_reader :project
-
- def set(klass_name, object_id, phabricator_id)
- key = cache_key_for_phabricator_id(phabricator_id)
-
- redis.with do |r|
- r.multi do |multi|
- multi.mapped_hmset(key,
- { classname: klass_name, database_id: object_id })
- multi.expire(key, timeout)
- end
- end
- end
-
- def get(phabricator_id)
- key = cache_key_for_phabricator_id(phabricator_id)
-
- redis.with do |r|
- r.pipelined do |pipe|
- # Extend the TTL when a key was
- pipe.expire(key, timeout)
- pipe.mapped_hmget(key, :classname, :database_id)
- end.last
- end
- end
-
- def cache_key_for_phabricator_id(phabricator_id)
- "#{Redis::Cache::CACHE_NAMESPACE}/phabricator-import/#{project.id}/#{phabricator_id}"
- end
-
- def redis
- Gitlab::Redis::Cache
- end
-
- def timeout
- # Setting the timeout to the same one as we do for clearing stuck jobs
- # this makes sure all cache is available while the import is running.
- Gitlab::Import::StuckImportJob::IMPORT_JOBS_EXPIRATION
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/phabricator_import/conduit.rb b/lib/gitlab/phabricator_import/conduit.rb
deleted file mode 100644
index 4c64d737389..00000000000
--- a/lib/gitlab/phabricator_import/conduit.rb
+++ /dev/null
@@ -1,9 +0,0 @@
-# frozen_string_literal: true
-module Gitlab
- module PhabricatorImport
- module Conduit
- ApiError = Class.new(Gitlab::PhabricatorImport::BaseError)
- ResponseError = Class.new(ApiError)
- end
- end
-end
diff --git a/lib/gitlab/phabricator_import/conduit/client.rb b/lib/gitlab/phabricator_import/conduit/client.rb
deleted file mode 100644
index 5945cde9618..00000000000
--- a/lib/gitlab/phabricator_import/conduit/client.rb
+++ /dev/null
@@ -1,41 +0,0 @@
-# frozen_string_literal: true
-module Gitlab
- module PhabricatorImport
- module Conduit
- class Client
- def initialize(phabricator_url, api_token)
- @phabricator_url = phabricator_url
- @api_token = api_token
- end
-
- def get(path, params: {})
- response = Gitlab::HTTP.get(build_url(path), body: build_params(params), headers: headers)
- Response.parse!(response)
- rescue *Gitlab::HTTP::HTTP_ERRORS => e
- # Wrap all errors from the API into an API-error.
- raise ApiError, e
- end
-
- private
-
- attr_reader :phabricator_url, :api_token
-
- def headers
- { "Accept" => 'application/json' }
- end
-
- def build_url(path)
- URI.join(phabricator_url, '/api/', path).to_s
- end
-
- def build_params(params)
- params = params.dup
- params.compact!
- params.reverse_merge!("api.token" => api_token)
-
- CGI.unescape(params.to_query)
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/phabricator_import/conduit/maniphest.rb b/lib/gitlab/phabricator_import/conduit/maniphest.rb
deleted file mode 100644
index 848b71e49e7..00000000000
--- a/lib/gitlab/phabricator_import/conduit/maniphest.rb
+++ /dev/null
@@ -1,28 +0,0 @@
-# frozen_string_literal: true
-module Gitlab
- module PhabricatorImport
- module Conduit
- class Maniphest
- def initialize(phabricator_url:, api_token:)
- @client = Client.new(phabricator_url, api_token)
- end
-
- def tasks(after: nil)
- TasksResponse.new(get_tasks(after))
- end
-
- private
-
- def get_tasks(after)
- client.get('maniphest.search',
- params: {
- after: after,
- attachments: { projects: 1, subscribers: 1, columns: 1 }
- })
- end
-
- attr_reader :client
- end
- end
- end
-end
diff --git a/lib/gitlab/phabricator_import/conduit/pagination.rb b/lib/gitlab/phabricator_import/conduit/pagination.rb
deleted file mode 100644
index 5f54cccdbc8..00000000000
--- a/lib/gitlab/phabricator_import/conduit/pagination.rb
+++ /dev/null
@@ -1,24 +0,0 @@
-# frozen_string_literal: true
-module Gitlab
- module PhabricatorImport
- module Conduit
- class Pagination
- def initialize(cursor_json)
- @cursor_json = cursor_json
- end
-
- def has_next_page?
- next_page.present?
- end
-
- def next_page
- cursor_json["after"]
- end
-
- private
-
- attr_reader :cursor_json
- end
- end
- end
-end
diff --git a/lib/gitlab/phabricator_import/conduit/response.rb b/lib/gitlab/phabricator_import/conduit/response.rb
deleted file mode 100644
index 26037ba183e..00000000000
--- a/lib/gitlab/phabricator_import/conduit/response.rb
+++ /dev/null
@@ -1,60 +0,0 @@
-# frozen_string_literal: true
-module Gitlab
- module PhabricatorImport
- module Conduit
- class Response
- def self.parse!(http_response)
- unless http_response.success?
- raise Gitlab::PhabricatorImport::Conduit::ResponseError,
- "Phabricator responded with #{http_response.status}"
- end
-
- response = new(Gitlab::Json.parse(http_response.body))
-
- unless response.success?
- raise ResponseError,
- "Phabricator Error: #{response.error_code}: #{response.error_info}"
- end
-
- response
- rescue JSON::JSONError => e
- raise ResponseError, e
- end
-
- def initialize(json)
- @json = json
- end
-
- def success?
- error_code.nil?
- end
-
- def error_code
- json['error_code']
- end
-
- def error_info
- json['error_info']
- end
-
- def data
- json_result&.fetch('data')
- end
-
- def pagination
- return unless cursor_info = json_result&.fetch('cursor')
-
- @pagination ||= Pagination.new(cursor_info)
- end
-
- private
-
- attr_reader :json
-
- def json_result
- json['result']
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/phabricator_import/conduit/tasks_response.rb b/lib/gitlab/phabricator_import/conduit/tasks_response.rb
deleted file mode 100644
index cbcf7259fb2..00000000000
--- a/lib/gitlab/phabricator_import/conduit/tasks_response.rb
+++ /dev/null
@@ -1,24 +0,0 @@
-# frozen_string_literal: true
-module Gitlab
- module PhabricatorImport
- module Conduit
- class TasksResponse
- def initialize(conduit_response)
- @conduit_response = conduit_response
- end
-
- delegate :pagination, to: :conduit_response
-
- def tasks
- @tasks ||= conduit_response.data.map do |task_json|
- Gitlab::PhabricatorImport::Representation::Task.new(task_json)
- end
- end
-
- private
-
- attr_reader :conduit_response
- end
- end
- end
-end
diff --git a/lib/gitlab/phabricator_import/conduit/user.rb b/lib/gitlab/phabricator_import/conduit/user.rb
deleted file mode 100644
index fc8c3f7cde9..00000000000
--- a/lib/gitlab/phabricator_import/conduit/user.rb
+++ /dev/null
@@ -1,31 +0,0 @@
-# frozen_string_literal: true
-module Gitlab
- module PhabricatorImport
- module Conduit
- class User
- MAX_PAGE_SIZE = 100
-
- def initialize(phabricator_url:, api_token:)
- @client = Client.new(phabricator_url, api_token)
- end
-
- def users(phids)
- phids.each_slice(MAX_PAGE_SIZE).map { |limited_phids| get_page(limited_phids) }
- end
-
- private
-
- def get_page(phids)
- UsersResponse.new(get_users(phids))
- end
-
- def get_users(phids)
- client.get('user.search',
- params: { constraints: { phids: phids } })
- end
-
- attr_reader :client
- end
- end
- end
-end
diff --git a/lib/gitlab/phabricator_import/conduit/users_response.rb b/lib/gitlab/phabricator_import/conduit/users_response.rb
deleted file mode 100644
index 3dfb29a7be5..00000000000
--- a/lib/gitlab/phabricator_import/conduit/users_response.rb
+++ /dev/null
@@ -1,23 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module PhabricatorImport
- module Conduit
- class UsersResponse
- def initialize(conduit_response)
- @conduit_response = conduit_response
- end
-
- def users
- @users ||= conduit_response.data.map do |user_json|
- Gitlab::PhabricatorImport::Representation::User.new(user_json)
- end
- end
-
- private
-
- attr_reader :conduit_response
- end
- end
- end
-end
diff --git a/lib/gitlab/phabricator_import/importer.rb b/lib/gitlab/phabricator_import/importer.rb
deleted file mode 100644
index 0666fa0df01..00000000000
--- a/lib/gitlab/phabricator_import/importer.rb
+++ /dev/null
@@ -1,44 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module PhabricatorImport
- class Importer
- def self.async?
- true
- end
-
- def self.imports_repository?
- # This does not really import a repository, but we want to skip all
- # repository related tasks in the `Projects::ImportService`
- true
- end
-
- def initialize(project)
- @project = project
- end
-
- def execute
- Gitlab::Import::SetAsyncJid.set_jid(project.import_state)
- schedule_first_tasks_page
-
- true
- rescue StandardError => e
- fail_import(e.message)
-
- false
- end
-
- private
-
- attr_reader :project
-
- def schedule_first_tasks_page
- ImportTasksWorker.schedule(project.id)
- end
-
- def fail_import(message)
- project.import_state.mark_as_failed(message)
- end
- end
- end
-end
diff --git a/lib/gitlab/phabricator_import/issues/importer.rb b/lib/gitlab/phabricator_import/issues/importer.rb
deleted file mode 100644
index 478c26af030..00000000000
--- a/lib/gitlab/phabricator_import/issues/importer.rb
+++ /dev/null
@@ -1,43 +0,0 @@
-# frozen_string_literal: true
-module Gitlab
- module PhabricatorImport
- module Issues
- class Importer
- def initialize(project, after = nil)
- @project = project
- @after = after
- end
-
- def execute
- schedule_next_batch
-
- tasks_response.tasks.each do |task|
- TaskImporter.new(project, task).execute
- end
- end
-
- private
-
- attr_reader :project, :after
-
- def schedule_next_batch
- return unless tasks_response.pagination.has_next_page?
-
- Gitlab::PhabricatorImport::ImportTasksWorker
- .schedule(project.id, tasks_response.pagination.next_page)
- end
-
- def tasks_response
- @tasks_response ||= client.tasks(after: after)
- end
-
- def client
- @client ||=
- Gitlab::PhabricatorImport::Conduit::Maniphest
- .new(phabricator_url: project.import_data.data['phabricator_url'],
- api_token: project.import_data.credentials[:api_token])
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/phabricator_import/issues/task_importer.rb b/lib/gitlab/phabricator_import/issues/task_importer.rb
deleted file mode 100644
index 9c419ecb700..00000000000
--- a/lib/gitlab/phabricator_import/issues/task_importer.rb
+++ /dev/null
@@ -1,61 +0,0 @@
-# frozen_string_literal: true
-module Gitlab
- module PhabricatorImport
- module Issues
- class TaskImporter
- def initialize(project, task)
- @project = project
- @task = task
- end
-
- def execute
- issue.author = user_finder.find(task.author_phid) || User.ghost
-
- # TODO: Reformat the description with attachments, escaping accidental
- # links and add attachments
- # https://gitlab.com/gitlab-org/gitlab-foss/issues/60603
- issue.assign_attributes(task.issue_attributes)
-
- save!
-
- if owner = user_finder.find(task.owner_phid)
- issue.assignees << owner
- end
-
- issue
- end
-
- private
-
- attr_reader :project, :task
-
- def save!
- # Just avoiding an extra redis call, we've already updated the expiry
- # when reading the id from the map
- was_persisted = issue.persisted?
-
- issue.save! if issue.changed?
-
- object_map.set_gitlab_model(issue, task.phabricator_id) unless was_persisted
- end
-
- def issue
- @issue ||= find_issue_by_phabricator_id(task.phabricator_id) ||
- project.issues.new
- end
-
- def user_finder
- @issue_finder ||= Gitlab::PhabricatorImport::UserFinder.new(project, task.phids)
- end
-
- def find_issue_by_phabricator_id(phabricator_id)
- object_map.get_gitlab_model(phabricator_id)
- end
-
- def object_map
- Gitlab::PhabricatorImport::Cache::Map.new(project)
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/phabricator_import/project_creator.rb b/lib/gitlab/phabricator_import/project_creator.rb
deleted file mode 100644
index 4de9eaa9500..00000000000
--- a/lib/gitlab/phabricator_import/project_creator.rb
+++ /dev/null
@@ -1,77 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module PhabricatorImport
- class ProjectCreator
- def initialize(current_user, params)
- @current_user = current_user
- @params = params.dup
- end
-
- def execute
- return unless import_url.present? && api_token.present?
-
- project = Projects::CreateService.new(current_user, create_params).execute
- return project unless project.persisted?
-
- project.project_feature.update!(project_feature_attributes)
-
- project
- end
-
- private
-
- attr_reader :current_user, :params
-
- def create_params
- {
- name: project_name,
- path: project_path,
- namespace_id: namespace_id,
- import_type: 'phabricator',
- import_url: Project::UNKNOWN_IMPORT_URL,
- import_data: import_data
- }
- end
-
- def project_name
- params[:name]
- end
-
- def project_path
- params[:path]
- end
-
- def namespace_id
- params[:namespace_id] || current_user.namespace_id
- end
-
- def import_url
- params[:phabricator_server_url]
- end
-
- def api_token
- params[:api_token]
- end
-
- def project_feature_attributes
- # everything disabled except for issues
- @project_features_attributes ||=
- ProjectFeature::FEATURES.to_h do |feature|
- [ProjectFeature.access_level_attribute(feature), ProjectFeature::DISABLED]
- end.merge(ProjectFeature.access_level_attribute(:issues) => ProjectFeature::ENABLED)
- end
-
- def import_data
- {
- data: {
- phabricator_url: import_url
- },
- credentials: {
- api_token: params.fetch(:api_token)
- }
- }
- end
- end
- end
-end
diff --git a/lib/gitlab/phabricator_import/representation/task.rb b/lib/gitlab/phabricator_import/representation/task.rb
deleted file mode 100644
index ba93fb37a8e..00000000000
--- a/lib/gitlab/phabricator_import/representation/task.rb
+++ /dev/null
@@ -1,72 +0,0 @@
-# frozen_string_literal: true
-module Gitlab
- module PhabricatorImport
- module Representation
- class Task
- def initialize(json)
- @json = json
- end
-
- def phabricator_id
- json['phid']
- end
-
- def author_phid
- json['fields']['authorPHID']
- end
-
- def owner_phid
- json['fields']['ownerPHID']
- end
-
- def phids
- @phids ||= [author_phid, owner_phid]
- end
-
- def issue_attributes
- @issue_attributes ||= {
- title: issue_title,
- description: issue_description,
- state: issue_state,
- created_at: issue_created_at,
- closed_at: issue_closed_at
- }
- end
-
- private
-
- attr_reader :json
-
- def issue_title
- # The 255 limit is the validation we impose on the Issue title in
- # Issuable
- @issue_title ||= json['fields']['name'].truncate(255)
- end
-
- def issue_description
- json['fields']['description']['raw']
- end
-
- def issue_state
- issue_closed_at.present? ? :closed : :opened
- end
-
- def issue_created_at
- return unless json['fields']['dateCreated']
-
- @issue_created_at ||= cast_datetime(json['fields']['dateCreated'])
- end
-
- def issue_closed_at
- return unless json['fields']['dateClosed']
-
- @issue_closed_at ||= cast_datetime(json['fields']['dateClosed'])
- end
-
- def cast_datetime(value)
- Time.at(value.to_i)
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/phabricator_import/representation/user.rb b/lib/gitlab/phabricator_import/representation/user.rb
deleted file mode 100644
index 7fd7cecc6ae..00000000000
--- a/lib/gitlab/phabricator_import/representation/user.rb
+++ /dev/null
@@ -1,25 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module PhabricatorImport
- module Representation
- class User
- def initialize(json)
- @json = json
- end
-
- def phabricator_id
- json['phid']
- end
-
- def username
- json['fields']['username']
- end
-
- private
-
- attr_reader :json
- end
- end
- end
-end
diff --git a/lib/gitlab/phabricator_import/user_finder.rb b/lib/gitlab/phabricator_import/user_finder.rb
deleted file mode 100644
index c6058d12527..00000000000
--- a/lib/gitlab/phabricator_import/user_finder.rb
+++ /dev/null
@@ -1,53 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module PhabricatorImport
- class UserFinder
- def initialize(project, phids)
- @project = project
- @phids = phids
- @loaded_phids = Set.new
- end
-
- def find(phid)
- found_user = object_map.get_gitlab_model(phid) do
- find_user_for_phid(phid)
- end
-
- loaded_phids << phid
-
- found_user
- end
-
- private
-
- attr_reader :project, :phids, :loaded_phids
-
- def object_map
- @object_map ||= Gitlab::PhabricatorImport::Cache::Map.new(project)
- end
-
- def find_user_for_phid(phid)
- phabricator_user = phabricator_users.find { |u| u.phabricator_id == phid }
- return unless phabricator_user
-
- project.authorized_users.find_by_username(phabricator_user.username)
- end
-
- def phabricator_users
- @user_responses ||= client.users(users_to_request).flat_map(&:users)
- end
-
- def users_to_request
- phids - loaded_phids.to_a
- end
-
- def client
- @client ||=
- Gitlab::PhabricatorImport::Conduit::User
- .new(phabricator_url: project.import_data.data['phabricator_url'],
- api_token: project.import_data.credentials[:api_token])
- end
- end
- end
-end
diff --git a/lib/gitlab/phabricator_import/worker_state.rb b/lib/gitlab/phabricator_import/worker_state.rb
deleted file mode 100644
index ffa2d3d7a43..00000000000
--- a/lib/gitlab/phabricator_import/worker_state.rb
+++ /dev/null
@@ -1,47 +0,0 @@
-# frozen_string_literal: true
-module Gitlab
- module PhabricatorImport
- class WorkerState
- def initialize(project_id)
- @project_id = project_id
- end
-
- def add_job
- redis.with do |r|
- r.pipelined do |pipe|
- pipe.incr(all_jobs_key)
- pipe.expire(all_jobs_key, timeout)
- end
- end
- end
-
- def remove_job
- redis.with do |r|
- r.decr(all_jobs_key)
- end
- end
-
- def running_count
- redis.with { |r| r.get(all_jobs_key) }.to_i
- end
-
- private
-
- attr_reader :project_id
-
- def redis
- Gitlab::Redis::SharedState
- end
-
- def all_jobs_key
- @all_jobs_key ||= "phabricator-import/jobs/project-#{project_id}/job-count"
- end
-
- def timeout
- # Make sure we get rid of all the information after a job is marked
- # as failed/succeeded
- Gitlab::Import::StuckImportJob::IMPORT_JOBS_EXPIRATION
- end
- end
- end
-end
diff --git a/lib/gitlab/private_commit_email.rb b/lib/gitlab/private_commit_email.rb
index 886c2c32d36..176402cadfe 100644
--- a/lib/gitlab/private_commit_email.rb
+++ b/lib/gitlab/private_commit_email.rb
@@ -19,7 +19,7 @@ module Gitlab
end
def user_ids_for_emails(emails)
- emails.map { |email| user_id_for_email(email) }.compact.uniq
+ emails.filter_map { |email| user_id_for_email(email) }.uniq
end
def for_user(user)
diff --git a/lib/gitlab/project_authorizations.rb b/lib/gitlab/project_authorizations.rb
index da3f67dde51..c770260a66e 100644
--- a/lib/gitlab/project_authorizations.rb
+++ b/lib/gitlab/project_authorizations.rb
@@ -12,7 +12,12 @@ module Gitlab
end
def calculate
- cte = recursive_cte
+ cte = if Feature.enabled?(:linear_project_authorization, user)
+ linear_cte
+ else
+ recursive_cte
+ end
+
cte_alias = cte.table.alias(Group.table_name)
projects = Project.arel_table
links = ProjectGroupLink.arel_table
@@ -47,11 +52,18 @@ module Gitlab
.where('p_ns.share_with_group_lock IS FALSE')
]
- ProjectAuthorization
- .unscoped
- .with
- .recursive(cte.to_arel)
- .select_from_union(relations)
+ if Feature.enabled?(:linear_project_authorization, user)
+ ProjectAuthorization
+ .unscoped
+ .with(cte.to_arel)
+ .select_from_union(relations)
+ else
+ ProjectAuthorization
+ .unscoped
+ .with
+ .recursive(cte.to_arel)
+ .select_from_union(relations)
+ end
end
private
@@ -89,6 +101,30 @@ module Gitlab
cte
end
+ def linear_cte
+ # Groups shared with user and their parent groups
+ shared_groups = Group
+ .select("namespaces.id, MAX(LEAST(members.access_level, group_group_links.group_access)) as access_level")
+ .joins("INNER JOIN group_group_links ON group_group_links.shared_group_id = namespaces.id
+ OR namespaces.traversal_ids @> ARRAY[group_group_links.shared_group_id::int]")
+ .joins("INNER JOIN members ON group_group_links.shared_with_group_id = members.source_id")
+ .merge(user.group_members)
+ .merge(GroupMember.active_state)
+ .group("namespaces.id")
+
+ # Groups the user is a member of and their parent groups.
+ lateral_query = Group.as_ids.where("namespaces.traversal_ids @> ARRAY [members.source_id]")
+ member_groups_with_ancestors = GroupMember.select("namespaces.id, MAX(members.access_level) as access_level")
+ .joins("CROSS JOIN LATERAL (#{lateral_query.to_sql}) as namespaces")
+ .group("namespaces.id")
+ .merge(user.group_members)
+ .merge(GroupMember.active_state)
+
+ union = Namespace.from_union([shared_groups, member_groups_with_ancestors])
+
+ Gitlab::SQL::CTE.new(:linear_namespaces_cte, union)
+ end
+
# Builds a LEFT JOIN to join optional memberships onto the CTE.
def join_members_on_namespaces
members = Member.arel_table
diff --git a/lib/gitlab/project_template.rb b/lib/gitlab/project_template.rb
index 5394cd115b1..c9ed4720e83 100644
--- a/lib/gitlab/project_template.rb
+++ b/lib/gitlab/project_template.rb
@@ -66,7 +66,6 @@ module Gitlab
ProjectTemplate.new('pelican', 'Pages/Pelican', _('Everything you need to create a GitLab Pages site using Pelican'), 'https://gitlab.com/pages/pelican', 'illustrations/third-party-logos/pelican.svg'),
ProjectTemplate.new('jekyll', 'Pages/Jekyll', _('Everything you need to create a GitLab Pages site using Jekyll'), 'https://gitlab.com/pages/jekyll', 'illustrations/logos/jekyll.svg'),
ProjectTemplate.new('plainhtml', 'Pages/Plain HTML', _('Everything you need to create a GitLab Pages site using plain HTML'), 'https://gitlab.com/pages/plain-html'),
- ProjectTemplate.new('gitbook', 'Pages/GitBook', _('Everything you need to create a GitLab Pages site using GitBook'), 'https://gitlab.com/pages/gitbook', 'illustrations/logos/gitbook.svg'),
ProjectTemplate.new('hexo', 'Pages/Hexo', _('Everything you need to create a GitLab Pages site using Hexo'), 'https://gitlab.com/pages/hexo', 'illustrations/logos/hexo.svg'),
ProjectTemplate.new('middleman', 'Pages/Middleman', _('Everything you need to create a GitLab Pages site using Middleman'), 'https://gitlab.com/gitlab-org/project-templates/middleman', 'illustrations/logos/middleman.svg'),
ProjectTemplate.new('gitpod_spring_petclinic', 'Gitpod/Spring Petclinic', _('A Gitpod configured Webapplication in Spring and Java'), 'https://gitlab.com/gitlab-org/project-templates/gitpod-spring-petclinic', 'illustrations/logos/gitpod.svg'),
@@ -81,8 +80,9 @@ module Gitlab
ProjectTemplate.new('jsonnet', 'Jsonnet for Dynamic Child Pipelines', _('An example showing how to use Jsonnet with GitLab dynamic child pipelines'), 'https://gitlab.com/gitlab-org/project-templates/jsonnet'),
ProjectTemplate.new('cluster_management', 'GitLab Cluster Management', _('An example project for managing Kubernetes clusters integrated with GitLab'), 'https://gitlab.com/gitlab-org/project-templates/cluster-management'),
ProjectTemplate.new('kotlin_native_linux', 'Kotlin Native Linux', _('A basic template for developing Linux programs using Kotlin Native'), 'https://gitlab.com/gitlab-org/project-templates/kotlin-native-linux'),
- ProjectTemplate.new('typo3_distribution', 'TYPO3 Distribution', _('A template for starting a new TYPO3 project'), 'https://gitlab.com/gitlab-org/project-templates/typo3-distribution', 'illustrations/logos/typo3.svg')
- ].freeze
+ ProjectTemplate.new('typo3_distribution', 'TYPO3 Distribution', _('A template for starting a new TYPO3 project'), 'https://gitlab.com/gitlab-org/project-templates/typo3-distribution', 'illustrations/logos/typo3.svg'),
+ ProjectTemplate.new('laravel', 'Laravel Framework', _('A basic folder structure of a Laravel application, to help you get started.'), 'https://gitlab.com/gitlab-org/project-templates/laravel', 'illustrations/logos/laravel.svg')
+ ]
end
# rubocop:enable Metrics/AbcSize
diff --git a/lib/gitlab/prometheus/internal.rb b/lib/gitlab/prometheus/internal.rb
index fe06b97add6..55e1837d101 100644
--- a/lib/gitlab/prometheus/internal.rb
+++ b/lib/gitlab/prometheus/internal.rb
@@ -27,7 +27,7 @@ module Gitlab
def self.server_address
Gitlab.config.prometheus.server_address.to_s if Gitlab.config.prometheus
- rescue Settingslogic::MissingSetting
+ rescue GitlabSettings::MissingSetting
Gitlab::AppLogger.error('Prometheus server_address is not present in config/gitlab.yml')
nil
@@ -35,7 +35,7 @@ module Gitlab
def self.prometheus_enabled?
Gitlab.config.prometheus.enabled if Gitlab.config.prometheus
- rescue Settingslogic::MissingSetting
+ rescue GitlabSettings::MissingSetting
Gitlab::AppLogger.error('prometheus.enabled is not present in config/gitlab.yml')
false
diff --git a/lib/gitlab/prometheus/queries/knative_invocation_query.rb b/lib/gitlab/prometheus/queries/knative_invocation_query.rb
deleted file mode 100644
index 6438995b576..00000000000
--- a/lib/gitlab/prometheus/queries/knative_invocation_query.rb
+++ /dev/null
@@ -1,42 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Prometheus
- module Queries
- class KnativeInvocationQuery < BaseQuery
- include QueryAdditionalMetrics
-
- def query(serverless_function_id)
- PrometheusMetricsFinder
- .new(identifier: :system_metrics_knative_function_invocation_count, common: true)
- .execute
- .first
- .to_query_metric
- .tap do |q|
- q.queries[0][:result] = run_query(q.queries[0][:query_range], context(serverless_function_id))
- end
- end
-
- protected
-
- def context(function_id)
- function = ::Serverless::Function.find_by_id(function_id)
- {
- function_name: function.name,
- kube_namespace: function.namespace
- }
- end
-
- def run_query(query, context)
- query %= context
- client_query_range(query, start_time: 8.hours.ago.to_f, end_time: Time.now.to_f)
- end
-
- def self.transform_reactive_result(result)
- result[:metrics] = result.delete :data
- result
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/puma_logging/json_formatter.rb b/lib/gitlab/puma_logging/json_formatter.rb
index 9eeb980fc53..6d97b6615aa 100644
--- a/lib/gitlab/puma_logging/json_formatter.rb
+++ b/lib/gitlab/puma_logging/json_formatter.rb
@@ -1,6 +1,7 @@
# frozen_string_literal: true
require 'json'
+require 'time'
module Gitlab
module PumaLogging
diff --git a/lib/gitlab/quick_actions/extractor.rb b/lib/gitlab/quick_actions/extractor.rb
index 2e4817e6b17..015dbe7063c 100644
--- a/lib/gitlab/quick_actions/extractor.rb
+++ b/lib/gitlab/quick_actions/extractor.rb
@@ -63,11 +63,12 @@ module Gitlab
#{CODE_REGEX} | #{INLINE_CODE_REGEX} | #{HTML_BLOCK_REGEX} | #{QUOTE_BLOCK_REGEX}
}mix.freeze
- attr_reader :command_definitions
+ attr_reader :command_definitions, :keep_actions
- def initialize(command_definitions)
+ def initialize(command_definitions, keep_actions: false)
@command_definitions = command_definitions
@commands_regex = {}
+ @keep_actions = keep_actions
end
# Extracts commands from content and return an array of commands.
@@ -76,8 +77,8 @@ module Gitlab
# ['command1'],
# ['command3', 'arg1 arg2'],
# ]
- # The command and the arguments are stripped.
- # The original command text is removed from the given `content`.
+ # The original command text and arguments are removed from the given `content`,
+ # unless `keep_actions` is true.
#
# Usage:
# ```
@@ -85,6 +86,11 @@ module Gitlab
# msg = %(hello\n/labels ~foo ~"bar baz"\nworld)
# commands = extractor.extract_commands(msg) #=> [['labels', '~foo ~"bar baz"']]
# msg #=> "hello\nworld"
+ #
+ # extractor = Gitlab::QuickActions::Extractor.new([:open, :assign, :labels], keep_actions: true)
+ # msg = %(hello\n/labels ~foo ~"bar baz"\nworld)
+ # commands = extractor.extract_commands(msg) #=> [['labels', '~foo ~"bar baz"']]
+ # msg #=> "hello\n/labels ~foo ~"bar baz"\n\nworld"
# ```
def extract_commands(content, only: nil)
return [content, []] unless content
@@ -138,6 +144,10 @@ module Gitlab
if redact
output = "`/#{matched_text[:cmd]}#{" " + matched_text[:arg] if matched_text[:arg]}`"
output += "\n" if matched_text[0].include?("\n")
+ elsif keep_actions
+ # requires an additional newline so that when rendered, it appears
+ # on its own line, rather than all on the same line
+ output = "\n#{matched_text[0]}\n"
end
end
diff --git a/lib/gitlab/quick_actions/issue_actions.rb b/lib/gitlab/quick_actions/issue_actions.rb
index ae8bc102f57..d7e9e1a980b 100644
--- a/lib/gitlab/quick_actions/issue_actions.rb
+++ b/lib/gitlab/quick_actions/issue_actions.rb
@@ -255,11 +255,12 @@ module Gitlab
execution_message { _('Issue has been promoted to incident') }
types Issue
condition do
- !quick_action_target.incident? &&
+ !quick_action_target.work_item_type&.incident? &&
current_user.can?(:"set_#{quick_action_target.issue_type}_metadata", quick_action_target)
end
command :promote_to_incident do
- @updates[:issue_type] = "incident"
+ @updates[:issue_type] = :incident
+ @updates[:work_item_type] = ::WorkItems::Type.default_by_type(:incident)
end
desc { _('Add customer relation contacts') }
@@ -297,7 +298,7 @@ module Gitlab
params '<timeline comment> | <date(YYYY-MM-DD)> <time(HH:MM)>'
types Issue
condition do
- quick_action_target.incident? &&
+ quick_action_target.work_item_type&.incident? &&
current_user.can?(:admin_incident_management_timeline_event, quick_action_target)
end
parse_params do |event_params|
diff --git a/lib/gitlab/quick_actions/merge_request_actions.rb b/lib/gitlab/quick_actions/merge_request_actions.rb
index f0ad6653c5e..c374593bf01 100644
--- a/lib/gitlab/quick_actions/merge_request_actions.rb
+++ b/lib/gitlab/quick_actions/merge_request_actions.rb
@@ -102,7 +102,7 @@ module Gitlab
(quick_action_target.new_record? || current_user.can?(:"update_#{quick_action_target.to_ability_name}", quick_action_target))
end
command :draft do
- @updates[:wip_event] = draft_action_command
+ @updates[:wip_event] = 'draft'
end
desc { _('Set the Ready status') }
@@ -309,21 +309,11 @@ module Gitlab
noun = quick_action_target.to_ability_name.humanize(capitalize: false)
if !quick_action_target.draft?
_("%{verb} this %{noun} as a draft.")
- elsif Feature.disabled?(:draft_quick_action_non_toggle, quick_action_target.project)
- _("%{verb} this %{noun} as ready.")
else
_("No change to this %{noun}'s draft status.")
end % { verb: verb, noun: noun }
end
- def draft_action_command
- if Feature.disabled?(:draft_quick_action_non_toggle, quick_action_target.project)
- quick_action_target.draft? ? 'ready' : 'draft'
- else
- 'draft'
- end
- end
-
def merge_orchestration_service
@merge_orchestration_service ||= ::MergeRequests::MergeOrchestrationService.new(project, current_user)
end
diff --git a/lib/gitlab/quick_actions/relate_actions.rb b/lib/gitlab/quick_actions/relate_actions.rb
index 4c8035f192e..b8cbfdefda1 100644
--- a/lib/gitlab/quick_actions/relate_actions.rb
+++ b/lib/gitlab/quick_actions/relate_actions.rb
@@ -8,19 +8,27 @@ module Gitlab
included do
desc { _('Mark this issue as related to another issue') }
- explanation do |related_reference|
- _('Marks this issue as related to %{issue_ref}.') % { issue_ref: related_reference }
+ explanation do |target_issues|
+ _('Marks this issue as related to %{issue_ref}.') % { issue_ref: target_issues.to_sentence }
end
- execution_message do |related_reference|
- _('Marked this issue as related to %{issue_ref}.') % { issue_ref: related_reference }
+ execution_message do |target_issues|
+ _('Marked this issue as related to %{issue_ref}.') % { issue_ref: target_issues.to_sentence }
end
- params '#issue'
+ params '<#issue | group/project#issue | issue URL>'
types Issue
- condition do
- current_user.can?(:"update_#{quick_action_target.to_ability_name}", quick_action_target)
+ condition { can_relate_issues? }
+ parse_params { |issues| format_params(issues) }
+ command :relate do |target_issues|
+ create_links(target_issues)
end
- command :relate do |related_reference|
- service = IssueLinks::CreateService.new(quick_action_target, current_user, { issuable_references: [related_reference] })
+
+ private
+
+ def create_links(references, type: 'relates_to')
+ service = IssueLinks::CreateService.new(
+ quick_action_target,
+ current_user, { issuable_references: references, link_type: type }
+ )
create_issue_link = proc { service.execute }
if quick_action_target.persisted?
@@ -29,6 +37,14 @@ module Gitlab
quick_action_target.run_after_commit(&create_issue_link)
end
end
+
+ def can_relate_issues?
+ current_user.can?(:admin_issue_link, quick_action_target)
+ end
+
+ def format_params(issue_references)
+ issue_references.split(' ')
+ end
end
end
end
diff --git a/lib/gitlab/quick_actions/work_item_actions.rb b/lib/gitlab/quick_actions/work_item_actions.rb
new file mode 100644
index 00000000000..fa43308c9e2
--- /dev/null
+++ b/lib/gitlab/quick_actions/work_item_actions.rb
@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module QuickActions
+ module WorkItemActions
+ extend ActiveSupport::Concern
+ include Gitlab::QuickActions::Dsl
+
+ included do
+ desc { _('Change work item type') }
+ explanation do |target_type|
+ format(_("Converts work item to %{type}. Widgets not supported in new type are removed."), type: target_type)
+ end
+ types WorkItem
+ condition do
+ quick_action_target&.project&.work_items_mvc_2_feature_flag_enabled?
+ end
+ params 'Task | Objective | Key Result | Issue'
+ command :type do |type_name|
+ work_item_type = ::WorkItems::Type.find_by_name(type_name)
+ errors = validate_type(work_item_type)
+
+ if errors.present?
+ @execution_message[:type] = errors
+ else
+ @updates[:issue_type] = work_item_type.base_type
+ @updates[:work_item_type] = work_item_type
+ @execution_message[:type] = _('Type changed successfully.')
+ end
+ end
+ end
+
+ private
+
+ def validate_type(type)
+ return type_error(:not_found) unless type.present?
+ return type_error(:same_type) if quick_action_target.work_item_type == type
+ return type_error(:forbidden) unless current_user.can?(:"create_#{type.base_type}", quick_action_target)
+
+ nil
+ end
+
+ def type_error(reason)
+ message = {
+ not_found: 'Provided type is not supported',
+ same_type: 'Types are the same',
+ forbidden: 'You have insufficient permissions'
+ }.freeze
+
+ format(_("Failed to convert this work item: %{reason}."), { reason: message[reason] })
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/rack_attack.rb b/lib/gitlab/rack_attack.rb
index bedbe9c0bff..d999b706d6c 100644
--- a/lib/gitlab/rack_attack.rb
+++ b/lib/gitlab/rack_attack.rb
@@ -19,7 +19,7 @@ module Gitlab
[429, { 'Content-Type' => 'text/plain' }.merge(throttled_headers), [Gitlab::Throttle.rate_limiting_response_text]]
end
- rack_attack.cache.store = Gitlab::RackAttack::InstrumentedCacheStore.new
+ rack_attack.cache.store = Gitlab::RackAttack::Store.new
# Configure the throttles
configure_throttles(rack_attack)
diff --git a/lib/gitlab/rack_attack/instrumented_cache_store.rb b/lib/gitlab/rack_attack/instrumented_cache_store.rb
deleted file mode 100644
index d8beb259fba..00000000000
--- a/lib/gitlab/rack_attack/instrumented_cache_store.rb
+++ /dev/null
@@ -1,33 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module RackAttack
- # This class is a proxy for all Redis calls made by RackAttack. All
- # the calls are instrumented, then redirected to the underlying
- # store (in `.store). This class instruments the standard interfaces
- # of ActiveRecord::Cache defined in
- # https://github.com/rails/rails/blob/v6.0.3.1/activesupport/lib/active_support/cache.rb#L315
- #
- # For more information, please see
- # https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/751
- class InstrumentedCacheStore
- NOTIFICATION_CHANNEL = 'redis.rack_attack'
-
- delegate :silence!, :mute, to: :@upstream_store
-
- def initialize(upstream_store: ::Gitlab::Redis::RateLimiting.cache_store, notifier: ActiveSupport::Notifications)
- @upstream_store = upstream_store
- @notifier = notifier
- end
-
- [:fetch, :read, :read_multi, :write_multi, :fetch_multi, :write, :delete,
- :exist?, :delete_matched, :increment, :decrement, :cleanup, :clear].each do |interface|
- define_method interface do |*args, **k_args, &block|
- @notifier.instrument(NOTIFICATION_CHANNEL, operation: interface) do
- @upstream_store.public_send(interface, *args, **k_args, &block) # rubocop:disable GitlabSecurity/PublicSend
- end
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/rack_attack/store.rb b/lib/gitlab/rack_attack/store.rb
new file mode 100644
index 00000000000..e4a1b022c32
--- /dev/null
+++ b/lib/gitlab/rack_attack/store.rb
@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module RackAttack
+ class Store
+ InvalidAmount = Class.new(StandardError)
+
+ # The increment method gets called very often. The implementation below
+ # aims to minimize the number of Redis calls we make.
+ def increment(key, amount = 1, options = {})
+ # Our code below that prevents calling EXPIRE after every INCR assumes
+ # we always increment by 1. This is true in Rack::Attack as of v6.6.1.
+ # This guard should alert us if Rack::Attack changes its behavior in a
+ # future version.
+ raise InvalidAmount unless amount == 1
+
+ with do |redis|
+ key = namespace(key)
+ new_value = redis.incr(key)
+ expires_in = options[:expires_in]
+ redis.expire(key, expires_in) if new_value == 1 && expires_in
+ new_value
+ end
+ end
+
+ def read(key, _options = {})
+ with { |redis| redis.get(namespace(key)) }
+ end
+
+ def write(key, value, options = {})
+ with { |redis| redis.set(namespace(key), value, ex: options[:expires_in]) }
+ end
+
+ def delete(key, _options = {})
+ with { |redis| redis.del(namespace(key)) }
+ end
+
+ private
+
+ def with(&block)
+ # rubocop: disable CodeReuse/ActiveRecord
+ Gitlab::Redis::RateLimiting.with(&block)
+ # rubocop: enable CodeReuse/ActiveRecord
+ rescue ::Redis::BaseConnectionError
+ # Following the example of
+ # https://github.com/rack/rack-attack/blob/v6.6.1/lib/rack/attack/store_proxy/redis_proxy.rb#L61-L65,
+ # do not raise an error if we cannot connect to Redis. If
+ # Redis::RateLimiting is unavailable it should not take the site down.
+ nil
+ end
+
+ def namespace(key)
+ "#{Gitlab::Redis::Cache::CACHE_NAMESPACE}:#{key}"
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/reactive_cache_set_cache.rb b/lib/gitlab/reactive_cache_set_cache.rb
index 2de3c07712f..dc13bb927e6 100644
--- a/lib/gitlab/reactive_cache_set_cache.rb
+++ b/lib/gitlab/reactive_cache_set_cache.rb
@@ -11,13 +11,19 @@ module Gitlab
end
def clear_cache!(key)
+ use_pipeline = ::Feature.enabled?(:use_pipeline_over_multikey)
+
with do |redis|
keys = read(key).map { |value| "#{cache_namespace}:#{value}" }
keys << cache_key(key)
Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do
redis.pipelined do |pipeline|
- keys.each_slice(1000) { |subset| pipeline.unlink(*subset) }
+ if use_pipeline
+ keys.each { |key| pipeline.unlink(key) }
+ else
+ keys.each_slice(1000) { |subset| pipeline.unlink(*subset) }
+ end
end
end
end
diff --git a/lib/gitlab/redis.rb b/lib/gitlab/redis.rb
index 4d15022cca5..64ca89c6bff 100644
--- a/lib/gitlab/redis.rb
+++ b/lib/gitlab/redis.rb
@@ -10,10 +10,10 @@ module Gitlab
ALL_CLASSES = [
Gitlab::Redis::Cache,
Gitlab::Redis::DbLoadBalancing,
+ Gitlab::Redis::FeatureFlag,
Gitlab::Redis::Queues,
Gitlab::Redis::RateLimiting,
Gitlab::Redis::RepositoryCache,
- Gitlab::Redis::ClusterRateLimiting,
Gitlab::Redis::Sessions,
Gitlab::Redis::SharedState,
Gitlab::Redis::TraceChunks
diff --git a/lib/gitlab/redis/cache.rb b/lib/gitlab/redis/cache.rb
index 647573e59fe..ba3af3e7a6f 100644
--- a/lib/gitlab/redis/cache.rb
+++ b/lib/gitlab/redis/cache.rb
@@ -2,16 +2,6 @@
module Gitlab
module Redis
- # Match signature in
- # https://github.com/rails/rails/blob/v6.1.7.2/activesupport/lib/active_support/cache/redis_cache_store.rb#L59
- ERROR_HANDLER = ->(method:, returning:, exception:) do
- Gitlab::ErrorTracking.log_exception(
- exception,
- method: method,
- returning: returning.inspect
- )
- end
-
class Cache < ::Gitlab::Redis::Wrapper
CACHE_NAMESPACE = 'cache:gitlab'
@@ -22,8 +12,7 @@ module Gitlab
redis: pool,
compress: Gitlab::Utils.to_boolean(ENV.fetch('ENABLE_REDIS_CACHE_COMPRESSION', '1')),
namespace: CACHE_NAMESPACE,
- expires_in: default_ttl_seconds,
- error_handler: ::Gitlab::Redis::ERROR_HANDLER
+ expires_in: default_ttl_seconds
}
end
diff --git a/lib/gitlab/redis/cluster_rate_limiting.rb b/lib/gitlab/redis/cluster_rate_limiting.rb
deleted file mode 100644
index e9d1e4f0c3f..00000000000
--- a/lib/gitlab/redis/cluster_rate_limiting.rb
+++ /dev/null
@@ -1,11 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Redis
- class ClusterRateLimiting < ::Gitlab::Redis::Wrapper
- def self.config_fallback
- Cache
- end
- end
- end
-end
diff --git a/lib/gitlab/redis/feature_flag.rb b/lib/gitlab/redis/feature_flag.rb
new file mode 100644
index 00000000000..441ff669035
--- /dev/null
+++ b/lib/gitlab/redis/feature_flag.rb
@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Redis
+ class FeatureFlag < ::Gitlab::Redis::Wrapper
+ FeatureFlagStore = Class.new(ActiveSupport::Cache::RedisCacheStore)
+
+ class << self
+ # The data we store on FeatureFlag is currently stored on Cache.
+ def config_fallback
+ Cache
+ end
+
+ def cache_store
+ @cache_store ||= FeatureFlagStore.new(
+ redis: pool,
+ compress: Gitlab::Utils.to_boolean(ENV.fetch('ENABLE_REDIS_CACHE_COMPRESSION', '1')),
+ namespace: Cache::CACHE_NAMESPACE,
+ expires_in: 1.hour
+ )
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/redis/multi_store.rb b/lib/gitlab/redis/multi_store.rb
index a102267d52b..9571e2f92e6 100644
--- a/lib/gitlab/redis/multi_store.rb
+++ b/lib/gitlab/redis/multi_store.rb
@@ -5,12 +5,6 @@ module Gitlab
class MultiStore
include Gitlab::Utils::StrongMemoize
- class ReadFromPrimaryError < StandardError
- def message
- 'Value not found on the redis primary store. Read from the redis secondary store successful.'
- end
- end
-
class PipelinedDiffError < StandardError
def initialize(result_primary, result_secondary)
@result_primary = result_primary
@@ -32,41 +26,33 @@ module Gitlab
attr_reader :primary_store, :secondary_store, :instance_name
- FAILED_TO_READ_ERROR_MESSAGE = 'Failed to read from the redis primary_store.'
+ FAILED_TO_READ_ERROR_MESSAGE = 'Failed to read from the redis default_store.'
FAILED_TO_WRITE_ERROR_MESSAGE = 'Failed to write to the redis primary_store.'
FAILED_TO_RUN_PIPELINE = 'Failed to execute pipeline on the redis primary_store.'
SKIP_LOG_METHOD_MISSING_FOR_COMMANDS = %i[info].freeze
- # For ENUMERATOR_CACHE_HIT_VALIDATOR and READ_CACHE_HIT_VALIDATOR,
- # we define procs to validate cache hit. The only other acceptable value is nil,
- # in the case of errors being raised.
- #
- # If a command has no empty response, set ->(val) { true }
- #
- # Ref: https://www.rubydoc.info/github/redis/redis-rb/Redis/Commands
- #
- READ_CACHE_HIT_VALIDATOR = {
- exists: ->(val) { val != 0 },
- exists?: ->(val) { val },
- get: ->(val) { !val.nil? },
- hexists: ->(val) { val },
- hget: ->(val) { !val.nil? },
- hgetall: ->(val) { val.is_a?(Hash) && !val.empty? },
- hlen: ->(val) { val != 0 },
- hmget: ->(val) { val.is_a?(Array) && !val.compact.empty? },
- hscan_each: ->(val) { val.is_a?(Enumerator) && !val.first.nil? },
- mapped_hmget: ->(val) { val.is_a?(Hash) && !val.compact.empty? },
- mget: ->(val) { val.is_a?(Array) && !val.compact.empty? },
- scan_each: ->(val) { val.is_a?(Enumerator) && !val.first.nil? },
- scard: ->(val) { val != 0 },
- sismember: ->(val) { val },
- smembers: ->(val) { val.is_a?(Array) && !val.empty? },
- sscan: ->(val) { val != ['0', []] },
- sscan_each: ->(val) { val.is_a?(Enumerator) && !val.first.nil? },
- ttl: ->(val) { val != 0 && val != -2 }, # ttl returns -2 if the key does not exist. See https://redis.io/commands/ttl/
- zscan_each: ->(val) { val.is_a?(Enumerator) && !val.first.nil? }
- }.freeze
+ READ_COMMANDS = %i[
+ exists
+ exists?
+ get
+ hexists
+ hget
+ hgetall
+ hlen
+ hmget
+ hscan_each
+ mapped_hmget
+ mget
+ scan_each
+ scard
+ sismember
+ smembers
+ sscan
+ sscan_each
+ ttl
+ zscan_each
+ ].freeze
WRITE_COMMANDS = %i[
del
@@ -111,7 +97,7 @@ module Gitlab
end
# rubocop:disable GitlabSecurity/PublicSend
- READ_CACHE_HIT_VALIDATOR.each_key do |name|
+ READ_COMMANDS.each do |name|
define_method(name) do |*args, **kwargs, &block|
if use_primary_and_secondary_stores?
read_command(name, *args, **kwargs, &block)
@@ -186,12 +172,6 @@ module Gitlab
@pipelined_command_error.increment(command: command_name, instance_name: instance_name)
end
- def increment_read_fallback_count(command_name)
- @read_fallback_counter ||= Gitlab::Metrics.counter(:gitlab_redis_multi_store_read_fallback_total,
- 'Client side Redis MultiStore reading fallback')
- @read_fallback_counter.increment(command: command_name, instance_name: instance_name)
- end
-
def increment_method_missing_count(command_name)
@method_missing_counter ||= Gitlab::Metrics.counter(:gitlab_redis_multi_store_method_missing_total,
'Client side Redis MultiStore method missing')
@@ -247,7 +227,7 @@ module Gitlab
if @instance
send_command(@instance, command_name, *args, **kwargs, &block)
else
- read_one_with_fallback(command_name, *args, **kwargs, &block)
+ read_from_default(command_name, *args, **kwargs, &block)
end
end
@@ -259,35 +239,12 @@ module Gitlab
end
end
- def read_one_with_fallback(command_name, *args, **kwargs, &block)
- begin
- value = send_command(default_store, command_name, *args, **kwargs, &block)
- rescue StandardError => e
- log_error(e, command_name,
- multi_store_error_message: FAILED_TO_READ_ERROR_MESSAGE)
- end
-
- return value if block.nil? && cache_hit?(command_name, value)
-
- fallback_read(command_name, *args, **kwargs, &block)
- end
-
- def cache_hit?(command, value)
- validator = READ_CACHE_HIT_VALIDATOR[command]
- return false unless validator
-
- !value.nil? && validator.call(value)
- end
-
- def fallback_read(command_name, *args, **kwargs, &block)
- value = send_command(fallback_store, command_name, *args, **kwargs, &block)
-
- if value
- log_error(ReadFromPrimaryError.new, command_name)
- increment_read_fallback_count(command_name)
- end
-
- value
+ def read_from_default(command_name, *args, **kwargs, &block)
+ send_command(default_store, command_name, *args, **kwargs, &block)
+ rescue StandardError => e
+ log_error(e, command_name,
+ multi_store_error_message: FAILED_TO_READ_ERROR_MESSAGE)
+ raise
end
def write_both(command_name, *args, **kwargs, &block)
diff --git a/lib/gitlab/redis/rate_limiting.rb b/lib/gitlab/redis/rate_limiting.rb
index 12710bafbea..74b4ca12d18 100644
--- a/lib/gitlab/redis/rate_limiting.rb
+++ b/lib/gitlab/redis/rate_limiting.rb
@@ -3,6 +3,9 @@
module Gitlab
module Redis
class RateLimiting < ::Gitlab::Redis::Wrapper
+ # We create a subclass only for the purpose of differentiating between different stores in cache metrics
+ RateLimitingStore = Class.new(ActiveSupport::Cache::RedisCacheStore)
+
class << self
# The data we store on RateLimiting used to be stored on Cache.
def config_fallback
@@ -10,20 +13,7 @@ module Gitlab
end
def cache_store
- @cache_store ||= ActiveSupport::Cache::RedisCacheStore.new(
- redis: pool,
- namespace: Cache::CACHE_NAMESPACE,
- error_handler: ::Gitlab::Redis::ERROR_HANDLER
- )
- end
-
- private
-
- def redis
- primary_store = ::Redis.new(::Gitlab::Redis::ClusterRateLimiting.params)
- secondary_store = ::Redis.new(params)
-
- MultiStore.new(primary_store, secondary_store, name.demodulize)
+ @cache_store ||= RateLimitingStore.new(redis: pool, namespace: Cache::CACHE_NAMESPACE)
end
end
end
diff --git a/lib/gitlab/redis/repository_cache.rb b/lib/gitlab/redis/repository_cache.rb
index 6c7bc8c41d5..966c6584aa5 100644
--- a/lib/gitlab/redis/repository_cache.rb
+++ b/lib/gitlab/redis/repository_cache.rb
@@ -3,6 +3,9 @@
module Gitlab
module Redis
class RepositoryCache < ::Gitlab::Redis::Wrapper
+ # We create a subclass only for the purpose of differentiating between different stores in cache metrics
+ RepositoryCacheStore = Class.new(ActiveSupport::Cache::RedisCacheStore)
+
class << self
# The data we store on RepositoryCache used to be stored on Cache.
def config_fallback
@@ -10,12 +13,11 @@ module Gitlab
end
def cache_store
- @cache_store ||= ActiveSupport::Cache::RedisCacheStore.new(
+ @cache_store ||= RepositoryCacheStore.new(
redis: pool,
compress: Gitlab::Utils.to_boolean(ENV.fetch('ENABLE_REDIS_CACHE_COMPRESSION', '1')),
namespace: Cache::CACHE_NAMESPACE,
- expires_in: Cache.default_ttl_seconds,
- error_handler: ::Gitlab::Redis::ERROR_HANDLER
+ expires_in: Cache.default_ttl_seconds
)
end
end
diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb
index c990655769c..45fe04835cc 100644
--- a/lib/gitlab/redis/wrapper.rb
+++ b/lib/gitlab/redis/wrapper.rb
@@ -55,15 +55,11 @@ module Gitlab
def config_file_name
[
# Instance specific config sources:
- ENV["GITLAB_REDIS_#{store_name.underscore.upcase}_CONFIG_FILE"],
config_file_path("redis.#{store_name.underscore}.yml"),
# The current Redis instance may have been split off from another one
# (e.g. TraceChunks was split off from SharedState).
- config_fallback&.config_file_name,
-
- # Global config sources:
- ENV['GITLAB_REDIS_CONFIG_FILE']
+ config_fallback&.config_file_name
].compact.first
end
@@ -160,35 +156,10 @@ module Gitlab
def raw_config_hash
config_data = fetch_config
- config_hash =
- if config_data
- config_data.is_a?(String) ? { url: config_data } : config_data.deep_symbolize_keys
- else
- { url: '' }
- end
-
- if config_hash[:url].blank? && config_hash[:cluster].blank?
- config_hash[:url] = legacy_fallback_urls[self.class.store_name] || legacy_fallback_urls[self.class.config_fallback.store_name]
- end
-
- config_hash
- end
+ return { url: '' } if config_data.nil?
+ return { url: config_data } if config_data.is_a?(String)
- # These URLs were defined for cache, queues, and shared_state in
- # code. They are used only when no config file exists at all for a
- # given instance. The configuration does not seem particularly
- # useful - it uses different ports on localhost - but we cannot
- # confidently delete it as we don't know if any instances rely on
- # this.
- #
- # DO NOT ADD new instances here. All new instances should define a
- # `.config_fallback`, which will then be used to look up this URL.
- def legacy_fallback_urls
- {
- 'Cache' => 'redis://localhost:6380',
- 'Queues' => 'redis://localhost:6381',
- 'SharedState' => 'redis://localhost:6382'
- }
+ config_data.deep_symbolize_keys
end
def fetch_config
diff --git a/lib/gitlab/reference_extractor.rb b/lib/gitlab/reference_extractor.rb
index 540394f04bd..783b68fac12 100644
--- a/lib/gitlab/reference_extractor.rb
+++ b/lib/gitlab/reference_extractor.rb
@@ -4,7 +4,7 @@ module Gitlab
# Extract possible GFM references from an arbitrary String for further processing.
class ReferenceExtractor < Banzai::ReferenceExtractor
REFERABLES = %i(user issue label milestone mentioned_user mentioned_group mentioned_project
- merge_request snippet commit commit_range directly_addressed_user epic iteration vulnerability
+ merge_request snippet commit commit_range directly_addressed_user epic vulnerability
alert).freeze
attr_accessor :project, :current_user, :author
@@ -64,18 +64,24 @@ module Gitlab
end
def all
- REFERABLES.each { |referable| send(referable.to_s.pluralize) } # rubocop:disable GitlabSecurity/PublicSend
+ self.class.referrables.each { |referable| send(referable.to_s.pluralize) } # rubocop:disable GitlabSecurity/PublicSend
@references.values.flatten
end
- def self.references_pattern
- return @pattern if @pattern
+ class << self
+ def references_pattern
+ return @pattern if @pattern
- patterns = REFERABLES.map do |type|
- Banzai::ReferenceParser[type].reference_class.try(:reference_pattern)
- end.uniq
+ patterns = referrables.map do |type|
+ Banzai::ReferenceParser[type].reference_class.try(:reference_pattern)
+ end.uniq
- @pattern = Regexp.union(patterns.compact)
+ @pattern = Regexp.union(patterns.compact)
+ end
+
+ def referrables
+ @referrables ||= REFERABLES
+ end
end
private
@@ -90,3 +96,5 @@ module Gitlab
end
end
end
+
+Gitlab::ReferenceExtractor.prepend_mod
diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb
index 943218a9972..eb99805e2e8 100644
--- a/lib/gitlab/regex.rb
+++ b/lib/gitlab/regex.rb
@@ -5,7 +5,12 @@ module Gitlab
module Packages
CONAN_RECIPE_FILES = %w[conanfile.py conanmanifest.txt conan_sources.tgz conan_export.tgz].freeze
CONAN_PACKAGE_FILES = %w[conaninfo.txt conanmanifest.txt conan_package.tgz].freeze
+
PYPI_NORMALIZED_NAME_REGEX_STRING = '[-_.]+'
+
+ # see https://github.com/apache/maven/blob/c1dfb947b509e195c75d4275a113598cf3063c3e/maven-artifact/src/main/java/org/apache/maven/artifact/Artifact.java#L46
+ MAVEN_SNAPSHOT_DYNAMIC_PARTS = /\A.{0,1000}(-\d{8}\.\d{6}-\d+).{0,1000}\z/.freeze
+
API_PATH_REGEX = %r{^/api/v\d+/(projects/[^/]+/|groups?/[^/]+/-/)?packages/[A-Za-z]+}.freeze
def conan_package_reference_regex
@@ -141,7 +146,7 @@ module Gitlab
end
def debian_direct_upload_filename_regex
- @debian_direct_upload_filename_regex ||= %r{\A.*\.(deb|udeb)\z}o.freeze
+ @debian_direct_upload_filename_regex ||= %r{\A.*\.(deb|udeb|ddeb)\z}o.freeze
end
def helm_channel_regex
@@ -253,38 +258,45 @@ module Gitlab
end
end
- extend self
- extend Packages
+ module BulkImports
+ def bulk_import_destination_namespace_path_regex
+ # This regexp validates the string conforms to rules for a destination_namespace path:
+ # i.e does not start with a non-alphanumeric character,
+ # contains only alphanumeric characters, forward slashes, periods, and underscores,
+ # does not end with a period or forward slash, and has a relative path structure
+ # with no http protocol chars or leading or trailing forward slashes
+ # eg 'source/full/path' or 'destination_namespace' not 'https://example.com/destination/namespace/path'
+ # the regex also allows for an empty string ('') to be accepted as this is allowed in
+ # a bulk_import POST request
+ @bulk_import_destination_namespace_path_regex ||= %r/((\A\z)|(\A[0-9a-z]*(-_.)?[0-9a-z])(\/?[0-9a-z]*[-_.]?[0-9a-z])+\z)/i
+ end
- def bulk_import_destination_namespace_path_regex
- # This regexp validates the string conforms to rules for a destination_namespace path:
- # i.e does not start with a non-alphanumeric character except for periods or underscores,
- # contains only alphanumeric characters, forward slashes, periods, and underscores,
- # does not end with a period or forward slash, and has a relative path structure
- # with no http protocol chars or leading or trailing forward slashes
- # eg 'source/full/path' or 'destination_namespace' not 'https://example.com/destination/namespace/path'
- # the regex also allows for an empty string ('') to be accepted as this is allowed in
- # a bulk_import POST request
- @bulk_import_destination_namespace_path_regex ||= %r/((\A\z)|\A([.]?)[^\W](\/?[.]?[0-9a-z][-_]*)+\z)/i
- end
+ def bulk_import_source_full_path_regex
+ # This regexp validates the string conforms to rules for a source_full_path path:
+ # i.e does not start with a non-alphanumeric character except for periods or underscores,
+ # contains only alphanumeric characters, forward slashes, periods, and underscores,
+ # does not end with a period or forward slash, and has a relative path structure
+ # with no http protocol chars or leading or trailing forward slashes
+ # eg 'source/full/path' or 'destination_namespace' not 'https://example.com/source/full/path'
+ @bulk_import_source_full_path_regex ||= %r/\A([.]?)[^\W](\/?([-_.+]*)*[0-9a-z][-_]*)+\z/i
+ end
- def bulk_import_source_full_path_regex
- # This regexp validates the string conforms to rules for a source_full_path path:
- # i.e does not start with a non-alphanumeric character except for periods or underscores,
- # contains only alphanumeric characters, forward slashes, periods, and underscores,
- # does not end with a period or forward slash, and has a relative path structure
- # with no http protocol chars or leading or trailing forward slashes
- # eg 'source/full/path' or 'destination_namespace' not 'https://example.com/source/full/path'
- @bulk_import_source_full_path_regex ||= %r/\A([.]?)[^\W](\/?[.]?[0-9a-z][-_]*)+\z/i
- end
+ def bulk_import_source_full_path_regex_message
+ bulk_import_destination_namespace_path_regex_message
+ end
- def bulk_import_destination_namespace_path_regex_message
- "cannot start with a non-alphanumeric character except for periods or underscores, " \
- "can contain only alphanumeric characters, forward slashes, periods, and underscores, " \
- "cannot end with a period or forward slash, and has a relative path structure " \
- "with no http protocol chars or leading or trailing forward slashes" \
+ def bulk_import_destination_namespace_path_regex_message
+ "must have a relative path structure " \
+ "with no HTTP protocol characters, or leading or trailing forward slashes. " \
+ "Path segments must not start or end with a special character, " \
+ "and must not contain consecutive special characters."
+ end
end
+ extend self
+ extend Packages
+ extend BulkImports
+
def group_path_regex
# This regexp validates the string conforms to rules for a group slug:
# i.e does not start with a non-alphanumeric character except for periods or underscores,
@@ -297,7 +309,7 @@ module Gitlab
def group_path_regex_message
"cannot start with a non-alphanumeric character except for periods or underscores, " \
"can contain only alphanumeric characters, periods, and underscores, " \
- "cannot end with a period or forward slash, and has no leading or trailing forward slashes" \
+ "cannot end with a period or forward slash, and has no leading or trailing forward slashes." \
end
def project_name_regex
@@ -454,7 +466,7 @@ module Gitlab
# ```
MARKDOWN_CODE_BLOCK_REGEX_UNTRUSTED =
'(?P<code>' \
- '^```\n' \
+ '^```.*?\n' \
'(?:\n|.)*?' \
'\n```\ *$' \
')'.freeze
@@ -472,6 +484,17 @@ module Gitlab
)
}mx.freeze
+ # HTML block:
+ # <tag>
+ # Anything, including `>>>` blocks which are ignored by this filter
+ # </tag>
+ MARKDOWN_HTML_BLOCK_REGEX_UNTRUSTED =
+ '(?P<html>' \
+ '^<[^>]+?>\ *\n' \
+ '(?:\n|.)*?' \
+ '\n<\/[^>]+?>\ *$' \
+ ')'.freeze
+
# HTML comment line:
# <!-- some commented text -->
MARKDOWN_HTML_COMMENT_LINE_REGEX_UNTRUSTED =
@@ -494,6 +517,13 @@ module Gitlab
}mx.freeze
end
+ def markdown_code_or_html_blocks_untrusted
+ @markdown_code_or_html_blocks_untrusted ||=
+ "#{MARKDOWN_CODE_BLOCK_REGEX_UNTRUSTED}" \
+ "|" \
+ "#{MARKDOWN_HTML_BLOCK_REGEX_UNTRUSTED}"
+ end
+
def markdown_code_or_html_comments_untrusted
@markdown_code_or_html_comments_untrusted ||=
"#{MARKDOWN_CODE_BLOCK_REGEX_UNTRUSTED}" \
@@ -503,6 +533,17 @@ module Gitlab
"#{MARKDOWN_HTML_COMMENT_BLOCK_REGEX_UNTRUSTED}"
end
+ def markdown_code_or_html_blocks_or_html_comments_untrusted
+ @markdown_code_or_html_comments_untrusted ||=
+ "#{MARKDOWN_CODE_BLOCK_REGEX_UNTRUSTED}" \
+ "|" \
+ "#{MARKDOWN_HTML_BLOCK_REGEX_UNTRUSTED}" \
+ "|" \
+ "#{MARKDOWN_HTML_COMMENT_LINE_REGEX_UNTRUSTED}" \
+ "|" \
+ "#{MARKDOWN_HTML_COMMENT_BLOCK_REGEX_UNTRUSTED}"
+ end
+
# Based on Jira's project key format
# https://confluence.atlassian.com/adminjiraserver073/changing-the-project-key-format-861253229.html
# Avoids linking CVE IDs (https://cve.mitre.org/cve/identifiers/syntaxchange.html#new) as Jira issues.
@@ -550,11 +591,15 @@ module Gitlab
end
def issue
- @issue ||= /(?<issue>\d+)(?<format>\+)?(?=\W|\z)/
+ @issue ||= /(?<issue>\d+)(?<format>\+s{,1})?(?=\W|\z)/
+ end
+
+ def work_item
+ @work_item ||= /(?<work_item>\d+)(?<format>\+s{,1})?(?=\W|\z)/
end
def merge_request
- @merge_request ||= /(?<merge_request>\d+)(?<format>\+)?/
+ @merge_request ||= /(?<merge_request>\d+)(?<format>\+s{,1})?/
end
def base64_regex
diff --git a/lib/gitlab/registration_features/password_complexity.rb b/lib/gitlab/registration_features/password_complexity.rb
new file mode 100644
index 00000000000..6d165a7a665
--- /dev/null
+++ b/lib/gitlab/registration_features/password_complexity.rb
@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module RegistrationFeatures
+ class PasswordComplexity
+ def self.feature_available?
+ ::License.feature_available?(:password_complexity) ||
+ ::GitlabSubscriptions::Features.usage_ping_feature?(:password_complexity)
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/repository_size_error_message.rb b/lib/gitlab/repository_size_error_message.rb
index f5d82e61187..e7d527dd4ce 100644
--- a/lib/gitlab/repository_size_error_message.rb
+++ b/lib/gitlab/repository_size_error_message.rb
@@ -7,7 +7,8 @@ module Gitlab
delegate :current_size, :limit, :exceeded_size, :additional_repo_storage_available?, to: :@checker
# @param checker [RepositorySizeChecker]
- def initialize(checker)
+ def initialize(checker, message_params = {})
+ @message_params = message_params
@checker = checker
end
@@ -19,13 +20,21 @@ module Gitlab
"This merge request cannot be merged, #{base_message}"
end
+ def push_warning
+ _("##### WARNING ##### You have used %{usage_percentage} of the storage quota for %{namespace_name} " \
+ "(%{current_size} of %{size_limit}). If %{namespace_name} exceeds the storage quota, " \
+ "all projects in the namespace will be locked and actions will be restricted. " \
+ "To manage storage, or purchase additional storage, see %{manage_storage_url}. " \
+ "To learn more about restricted actions, see %{restricted_actions_url}") % push_message_params
+ end
+
def push_error(change_size = 0)
"Your push has been rejected, #{base_message(change_size)}. #{more_info_message}"
end
def new_changes_error
if additional_repo_storage_available?
- "Your push to this repository has been rejected because it would exceed storage limits. Please contact your GitLab administrator for more information."
+ "Your push to this repository has been rejected because it would exceed storage limits. #{more_info_message}"
else
"Your push to this repository would cause it to exceed the size limit of #{formatted(limit)} so it has been rejected. #{more_info_message}"
end
@@ -41,6 +50,19 @@ module Gitlab
private
+ attr_reader :message_params
+
+ def push_message_params
+ {
+ namespace_name: message_params[:namespace_name],
+ manage_storage_url: help_page_url('user/usage_quotas', 'manage-your-storage-usage'),
+ restricted_actions_url: help_page_url('user/read_only_namespaces', 'restricted-actions'),
+ current_size: formatted(current_size),
+ size_limit: formatted(limit),
+ usage_percentage: usage_percentage
+ }
+ end
+
def base_message(change_size = 0)
"because this repository has exceeded its size limit of #{formatted(limit)} by #{formatted(exceeded_size(change_size))}"
end
@@ -48,5 +70,13 @@ module Gitlab
def formatted(number)
number_to_human_size(number, delimiter: ',', precision: 2)
end
+
+ def usage_percentage
+ number_to_percentage(@checker.usage_ratio * 100, precision: 0)
+ end
+
+ def help_page_url(path, anchor = nil)
+ ::Gitlab::Routing.url_helpers.help_page_url(path, anchor: anchor)
+ end
end
end
diff --git a/lib/gitlab/request_context.rb b/lib/gitlab/request_context.rb
index c9eefe9a647..813468ece90 100644
--- a/lib/gitlab/request_context.rb
+++ b/lib/gitlab/request_context.rb
@@ -7,12 +7,28 @@ module Gitlab
RequestDeadlineExceeded = Class.new(StandardError)
- attr_accessor :client_ip, :start_thread_cpu_time, :request_start_time, :thread_memory_allocations
+ attr_accessor :client_ip, :spam_params, :start_thread_cpu_time, :request_start_time, :thread_memory_allocations
class << self
def instance
Gitlab::SafeRequestStore[:request_context] ||= new
end
+
+ def start_request_context(request:)
+ # We need to use Rack::Request to be consistent with Rails due to a Rails bug described in
+ # https://gitlab.com/gitlab-org/gitlab-foss/issues/58573#note_149799010
+ # Hosts behind a load balancer will only see 127.0.0.1 for the load balancer's IP.
+ rack_req = Rack::Request.new(request.env)
+ instance.client_ip = rack_req.ip
+
+ instance.spam_params = ::Spam::SpamParams.new_from_request(request: request)
+ instance.request_start_time = Gitlab::Metrics::System.real_time
+ end
+
+ def start_thread_context
+ instance.start_thread_cpu_time = Gitlab::Metrics::System.thread_cpu_time
+ instance.thread_memory_allocations = Gitlab::Memory::Instrumentation.start_thread_memory_allocations
+ end
end
def request_deadline
diff --git a/lib/gitlab/resource_events/assignment_event_recorder.rb b/lib/gitlab/resource_events/assignment_event_recorder.rb
new file mode 100644
index 00000000000..94bd05a17ba
--- /dev/null
+++ b/lib/gitlab/resource_events/assignment_event_recorder.rb
@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module ResourceEvents
+ class AssignmentEventRecorder
+ BATCH_SIZE = 100
+
+ def initialize(parent:, old_assignees:)
+ @parent = parent
+ @old_assignees = old_assignees
+ end
+
+ def record
+ return if Feature.disabled?(:record_issue_and_mr_assignee_events, parent.project)
+
+ case parent
+ when Issue
+ record_for_parent(
+ ::ResourceEvents::IssueAssignmentEvent,
+ :issue_id,
+ parent,
+ old_assignees
+ )
+ when MergeRequest
+ record_for_parent(
+ ::ResourceEvents::MergeRequestAssignmentEvent,
+ :merge_request_id,
+ parent,
+ old_assignees
+ )
+ end
+ end
+
+ private
+
+ attr_reader :parent, :old_assignees
+
+ def record_for_parent(resource_klass, foreign_key, parent, old_assignees)
+ removed_events = (old_assignees - parent.assignees).map do |unassigned_user|
+ {
+ foreign_key => parent.id,
+ user_id: unassigned_user.id,
+ action: :remove
+ }
+ end.to_set
+
+ added_events = (parent.assignees.to_a - old_assignees).map do |added_user|
+ {
+ foreign_key => parent.id,
+ user_id: added_user.id,
+ action: :add
+ }
+ end.to_set
+
+ (removed_events + added_events).each_slice(BATCH_SIZE) do |events|
+ resource_klass.insert_all(events)
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/runtime.rb b/lib/gitlab/runtime.rb
index 7e9fb82fb8b..f74f1489405 100644
--- a/lib/gitlab/runtime.rb
+++ b/lib/gitlab/runtime.rb
@@ -38,7 +38,7 @@ module Gitlab
end
def puma?
- !!defined?(::Puma)
+ !!defined?(::Puma::Server)
end
def sidekiq?
diff --git a/lib/gitlab/saas.rb b/lib/gitlab/saas.rb
index 16a7a697e6a..722475ce61d 100644
--- a/lib/gitlab/saas.rb
+++ b/lib/gitlab/saas.rb
@@ -49,6 +49,10 @@ module Gitlab
"https://about.gitlab.com/pricing#faq"
end
+ def self.about_feature_comparison_url
+ "https://about.gitlab.com/pricing/gitlab-com/feature-comparison"
+ end
+
def self.doc_url
'https://docs.gitlab.com'
end
diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb
index 37414f9e2b1..93befc2df57 100644
--- a/lib/gitlab/search_results.rb
+++ b/lib/gitlab/search_results.rb
@@ -120,6 +120,14 @@ module Gitlab
[]
end
+ def failed?
+ false
+ end
+
+ def error
+ nil
+ end
+
private
def collection_for(scope)
diff --git a/lib/gitlab/seeders/ci/runner/runner_fleet_seeder.rb b/lib/gitlab/seeders/ci/runner/runner_fleet_seeder.rb
index 082d267442c..23043fc8652 100644
--- a/lib/gitlab/seeders/ci/runner/runner_fleet_seeder.rb
+++ b/lib/gitlab/seeders/ci/runner/runner_fleet_seeder.rb
@@ -66,14 +66,18 @@ module Gitlab
plan_limits = Plan.default.actual_limits
if plan_limits.ci_registered_group_runners < @runner_count
- logger.error('The plan limits for group runners is set to ' \
+ warn 'The plan limits for group runners is set to ' \
"#{plan_limits.ci_registered_group_runners} runners. " \
- 'You should raise the plan limits to avoid errors during runner creation')
+ "You should raise the plan limits to avoid errors during runner creation by running " \
+ "the following command in the Rails console:\n" \
+ "Plan.default.actual_limits.update!(ci_registered_group_runners: #{@runner_count})"
return false
elsif plan_limits.ci_registered_project_runners < @runner_count
- logger.error('The plan limits for project runners is set to ' \
+ warn 'The plan limits for project runners is set to ' \
"#{plan_limits.ci_registered_project_runners} runners. " \
- 'You should raise the plan limits to avoid errors during runner creation')
+ "You should raise the plan limits to avoid errors during runner creation by running " \
+ "the following command in the Rails console:\n" \
+ "Plan.default.actual_limits.update!(ci_registered_project_runners: #{@runner_count})"
return false
end
@@ -205,7 +209,7 @@ module Gitlab
scope.runners_token
end
- response = ::Ci::Runners::RegisterRunnerService.new.execute(runners_token, name: name, **args)
+ response = ::Ci::Runners::RegisterRunnerService.new(runners_token, name: name, **args).execute
runner = response.payload[:runner]
::Ci::Runners::ProcessRunnerVersionUpdateWorker.new.perform(args[:version])
diff --git a/lib/gitlab/seeders/ci/variables_group_seeder.rb b/lib/gitlab/seeders/ci/variables_group_seeder.rb
new file mode 100644
index 00000000000..e5c5ee4b75f
--- /dev/null
+++ b/lib/gitlab/seeders/ci/variables_group_seeder.rb
@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Seeders
+ module Ci
+ class VariablesGroupSeeder
+ DEFAULT_SEED_COUNT = 10
+ DEFAULT_PREFIX = 'GROUP_VAR_'
+ DEFAULT_ENV = '*'
+
+ def initialize(params)
+ @group = Group.find_by_name(params[:name])
+ @seed_count = params[:seed_count] || DEFAULT_SEED_COUNT
+ @environment_scope = params[:environment_scope] || DEFAULT_ENV
+ @prefix = params[:prefix] || DEFAULT_PREFIX
+ end
+
+ def seed
+ if @group.nil?
+ warn 'ERROR: Group name is invalid.'
+ return
+ end
+
+ max_id = group.variables.maximum(:id).to_i
+ seed_count.times do
+ max_id += 1
+ create_ci_variable(max_id)
+ end
+ end
+
+ private
+
+ attr_reader :environment_scope, :group, :prefix, :seed_count
+
+ def create_ci_variable(id)
+ env = environment_scope == 'unique' ? "env_#{id}" : environment_scope
+ key = "#{prefix}#{id}"
+
+ if group.variables.by_environment_scope(env).find_by_key(key).present?
+ warn "WARNING: Group CI Variable with key '#{key}' already exists. Skipping to next CI variable..."
+ return
+ end
+
+ group.variables.create(
+ environment_scope: env,
+ key: key,
+ value: SecureRandom.hex(32)
+ )
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/seeders/ci/variables_instance_seeder.rb b/lib/gitlab/seeders/ci/variables_instance_seeder.rb
new file mode 100644
index 00000000000..d43defd192f
--- /dev/null
+++ b/lib/gitlab/seeders/ci/variables_instance_seeder.rb
@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Seeders
+ module Ci
+ class VariablesInstanceSeeder
+ DEFAULT_SEED_COUNT = 10
+ DEFAULT_PREFIX = 'INSTANCE_VAR_'
+
+ def initialize(params = {})
+ @seed_count = params[:seed_count] || DEFAULT_SEED_COUNT
+ @prefix = params[:prefix] || DEFAULT_PREFIX
+ end
+
+ def seed
+ max_id = ::Ci::InstanceVariable.maximum(:id).to_i
+ seed_count.times do
+ max_id += 1
+ create_ci_variable(max_id)
+ end
+ end
+
+ private
+
+ attr_reader :prefix, :seed_count
+
+ def create_ci_variable(id)
+ key = "#{prefix}#{id}"
+
+ if ::Ci::InstanceVariable.find_by_key(key)
+ warn "WARNING: Instance CI Variable with key '#{key}' already exists. Skipping to next CI variable..."
+ return
+ end
+
+ ::Ci::InstanceVariable.new(
+ key: key,
+ value: SecureRandom.hex(32)
+ ).save
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/seeders/ci/variables_project_seeder.rb b/lib/gitlab/seeders/ci/variables_project_seeder.rb
new file mode 100644
index 00000000000..c6b3dac7a4d
--- /dev/null
+++ b/lib/gitlab/seeders/ci/variables_project_seeder.rb
@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Seeders
+ module Ci
+ class VariablesProjectSeeder
+ DEFAULT_SEED_COUNT = 10
+ DEFAULT_PREFIX = 'VAR_'
+ DEFAULT_ENV = '*'
+
+ def initialize(params)
+ @project = Project.find_by_full_path(params[:project_path])
+ @seed_count = params[:seed_count] || DEFAULT_SEED_COUNT
+ @environment_scope = params[:environment_scope] || DEFAULT_ENV
+ @prefix = params[:prefix] || DEFAULT_PREFIX
+ end
+
+ def seed
+ if @project.nil?
+ warn 'ERROR: Project path is invalid.'
+ return
+ end
+
+ max_id = project.variables.maximum(:id).to_i
+ seed_count.times do
+ max_id += 1
+ create_ci_variable(max_id)
+ end
+ end
+
+ private
+
+ attr_reader :environment_scope, :prefix, :project, :seed_count
+
+ def create_ci_variable(id)
+ env = environment_scope == 'unique' ? "env_#{id}" : environment_scope
+ key = "#{prefix}#{id}"
+
+ if project.variables.by_environment_scope(env).find_by_key(key).present?
+ warn "WARNING: Project CI Variable with key '#{key}' already exists. Skipping to next CI variable..."
+ end
+
+ project.variables.create(
+ environment_scope: env,
+ key: key,
+ value: SecureRandom.hex(32)
+ )
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/seeders/project_environment_seeder.rb b/lib/gitlab/seeders/project_environment_seeder.rb
new file mode 100644
index 00000000000..3fc7d3d9b12
--- /dev/null
+++ b/lib/gitlab/seeders/project_environment_seeder.rb
@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Seeders
+ class ProjectEnvironmentSeeder
+ DEFAULT_SEED_COUNT = 10
+ DEFAULT_PREFIX = 'ENV_'
+
+ def initialize(params)
+ @project = Project.find_by_full_path(params[:project_path])
+ @seed_count = params[:seed_count] || DEFAULT_SEED_COUNT
+ @prefix = params[:prefix] || DEFAULT_PREFIX
+ end
+
+ def seed
+ if @project.nil?
+ warn 'ERROR: Project path is invalid.'
+ return
+ end
+
+ max_id = project.environments.maximum(:id).to_i
+ seed_count.times do
+ max_id += 1
+ create_project_environment_scope(max_id)
+ end
+ end
+
+ private
+
+ attr_reader :project, :seed_count, :prefix
+
+ def create_project_environment_scope(id)
+ name = "#{prefix}#{id}"
+
+ if project.environments.find_by_name(name).present?
+ warn "WARNING: Project Environment '#{name}' already exists. Skipping to next CI variable..."
+ return
+ end
+
+ project.environments.create(name: name)
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/serializer/ci/variables.rb b/lib/gitlab/serializer/ci/variables.rb
index 9abf3a54f37..a12bda0e5a7 100644
--- a/lib/gitlab/serializer/ci/variables.rb
+++ b/lib/gitlab/serializer/ci/variables.rb
@@ -12,7 +12,7 @@ module Gitlab
def load(string)
return unless string
- object = YAML.safe_load(string, [Symbol])
+ object = YAML.safe_load(string, permitted_classes: [Symbol])
object.map do |variable|
variable.symbolize_keys.tap do |variable|
diff --git a/lib/gitlab/serverless/service.rb b/lib/gitlab/serverless/service.rb
deleted file mode 100644
index c3ab2e9ddeb..00000000000
--- a/lib/gitlab/serverless/service.rb
+++ /dev/null
@@ -1,102 +0,0 @@
-# frozen_string_literal: true
-
-class Gitlab::Serverless::Service
- include Gitlab::Utils::StrongMemoize
-
- def initialize(attributes)
- @attributes = attributes
- end
-
- def name
- @attributes.dig('metadata', 'name')
- end
-
- def namespace
- @attributes.dig('metadata', 'namespace')
- end
-
- def environment_scope
- @attributes.dig('environment_scope')
- end
-
- def environment
- @attributes.dig('environment')
- end
-
- def podcount
- @attributes.dig('podcount')
- end
-
- def created_at
- strong_memoize(:created_at) do
- timestamp = @attributes.dig('metadata', 'creationTimestamp')
- DateTime.parse(timestamp) if timestamp
- end
- end
-
- def image
- @attributes.dig(
- 'spec',
- 'runLatest',
- 'configuration',
- 'build',
- 'template',
- 'name')
- end
-
- def description
- knative_07_description || knative_05_06_description
- end
-
- def cluster
- @attributes.dig('cluster')
- end
-
- def url
- proxy_url || knative_06_07_url || knative_05_url
- end
-
- private
-
- def proxy_url
- if cluster&.serverless_domain
- ::Serverless::Domain.new(
- function_name: name,
- serverless_domain_cluster: cluster.serverless_domain,
- environment: environment
- ).uri.to_s
- end
- end
-
- def knative_07_description
- @attributes.dig(
- 'spec',
- 'template',
- 'metadata',
- 'annotations',
- 'Description'
- )
- end
-
- def knative_05_06_description
- @attributes.dig(
- 'spec',
- 'runLatest',
- 'configuration',
- 'revisionTemplate',
- 'metadata',
- 'annotations',
- 'Description')
- end
-
- def knative_05_url
- domain = @attributes.dig('status', 'domain')
- return unless domain
-
- "http://#{domain}"
- end
-
- def knative_06_07_url
- @attributes.dig('status', 'url')
- end
-end
diff --git a/lib/gitlab/service_desk.rb b/lib/gitlab/service_desk.rb
index b3d6e890e03..5acbde552c8 100644
--- a/lib/gitlab/service_desk.rb
+++ b/lib/gitlab/service_desk.rb
@@ -10,7 +10,7 @@ module Gitlab
end
def self.supported?
- Gitlab::IncomingEmail.enabled? && Gitlab::IncomingEmail.supports_wildcard?
+ Gitlab::Email::IncomingEmail.enabled? && Gitlab::Email::IncomingEmail.supports_wildcard?
end
end
end
diff --git a/lib/gitlab/service_desk_email.rb b/lib/gitlab/service_desk_email.rb
deleted file mode 100644
index bc49efafdda..00000000000
--- a/lib/gitlab/service_desk_email.rb
+++ /dev/null
@@ -1,26 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module ServiceDeskEmail
- class << self
- include Gitlab::Email::Common
-
- def config
- Gitlab.config.service_desk_email
- end
-
- def key_from_address(address)
- wildcard_address = config&.address
- return unless wildcard_address
-
- Gitlab::IncomingEmail.key_from_address(address, wildcard_address: wildcard_address)
- end
-
- def address_for_key(key)
- return if config.address.blank?
-
- config.address.sub(WILDCARD_PLACEHOLDER, key)
- end
- end
- end
-end
diff --git a/lib/gitlab/set_cache.rb b/lib/gitlab/set_cache.rb
index 3d2ff5a68d2..623b254c4e0 100644
--- a/lib/gitlab/set_cache.rb
+++ b/lib/gitlab/set_cache.rb
@@ -22,7 +22,13 @@ module Gitlab
keys_to_expire = keys.map { |key| cache_key(key) }
Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do
- redis.unlink(*keys_to_expire)
+ if ::Feature.enabled?(:use_pipeline_over_multikey)
+ redis.pipelined do |pipeline|
+ keys_to_expire.each { |key| pipeline.unlink(key) }
+ end.sum
+ else
+ redis.unlink(*keys_to_expire)
+ end
end
end
end
diff --git a/lib/gitlab/setup_helper.rb b/lib/gitlab/setup_helper.rb
index 1e42003b203..2e09a4fce12 100644
--- a/lib/gitlab/setup_helper.rb
+++ b/lib/gitlab/setup_helper.rb
@@ -80,7 +80,7 @@ module Gitlab
# because it uses a Unix socket.
# For development and testing purposes, an extra storage is added to gitaly,
# which is not known to Rails, but must be explicitly stubbed.
- def configuration_toml(gitaly_dir, storage_paths, options, gitaly_ruby: true)
+ def configuration_toml(gitaly_dir, storage_paths, options)
storages = []
address = nil
@@ -128,7 +128,6 @@ module Gitlab
FileUtils.mkdir(runtime_dir) unless File.exist?(runtime_dir)
config[:runtime_dir] = runtime_dir
- config[:'gitaly-ruby'] = { dir: File.join(gitaly_dir, 'ruby') } if gitaly_ruby
config[:'gitlab-shell'] = { dir: Gitlab.config.gitlab_shell.path }
config[:bin_dir] = File.expand_path(File.join(gitaly_dir, '_build', 'bin')) # binaries by default are in `_build/bin`
config[:gitlab] = { url: Gitlab.config.gitlab.url }
diff --git a/lib/gitlab/sidekiq_config.rb b/lib/gitlab/sidekiq_config.rb
index 7e2a934b3dd..33a15d95d22 100644
--- a/lib/gitlab/sidekiq_config.rb
+++ b/lib/gitlab/sidekiq_config.rb
@@ -57,13 +57,12 @@ module Gitlab
@cron_jobs ||= begin
Gitlab.config.load_dynamic_cron_schedules!
- # Load recurring jobs from gitlab.yml
- # UGLY Hack to get nested hash from settingslogic
- jobs = Gitlab::Json.parse(Gitlab.config.cron_jobs.to_json)
+ jobs = Gitlab.config.cron_jobs.to_hash
jobs.delete('poll_interval') # Would be interpreted as a job otherwise
- # UGLY hack: Settingslogic doesn't allow 'class' key
+ # Settingslogic (former gem used for yaml configuration) didn't allow 'class' key
+ # Therefore, we configure cron jobs with `job_class` as a workaround.
required_keys = %w[job_class cron]
jobs.each do |k, v|
if jobs[k] && required_keys.all? { |s| jobs[k].key?(s) }
diff --git a/lib/gitlab/sidekiq_config/worker_router.rb b/lib/gitlab/sidekiq_config/worker_router.rb
index 0670e5521df..6d5ecb64065 100644
--- a/lib/gitlab/sidekiq_config/worker_router.rb
+++ b/lib/gitlab/sidekiq_config/worker_router.rb
@@ -77,6 +77,11 @@ module Gitlab
def parse_routing_rules(routing_rules)
raise InvalidRoutingRuleError, 'The set of routing rule must be an array' unless routing_rules.is_a?(Array)
+ unless routing_rules.last&.first == WorkerMatcher::WILDCARD_MATCH
+ Gitlab::AppLogger.warn "sidekiq.routing_rules config is missing a catch-all `*` entry as the last rule. " \
+ "Consider adding `[['*', 'default']]` at the end of routing_rules."
+ end
+
routing_rules.map do |rule_tuple|
raise InvalidRoutingRuleError, "Routing rule `#{rule_tuple.inspect}` is invalid" unless valid_routing_rule?(rule_tuple)
diff --git a/lib/gitlab/sidekiq_daemon/memory_killer.rb b/lib/gitlab/sidekiq_daemon/memory_killer.rb
deleted file mode 100644
index 1682d62d782..00000000000
--- a/lib/gitlab/sidekiq_daemon/memory_killer.rb
+++ /dev/null
@@ -1,293 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module SidekiqDaemon
- class MemoryKiller < Daemon
- include ::Gitlab::Utils::StrongMemoize
-
- # Today 64-bit CPU support max 256T memory. It is big enough.
- MAX_MEMORY_KB = 256 * 1024 * 1024 * 1024
- # RSS below `soft_limit_rss` is considered safe
- SOFT_LIMIT_RSS_KB = ENV.fetch('SIDEKIQ_MEMORY_KILLER_MAX_RSS', 2000000).to_i
- # RSS above `hard_limit_rss` will be stopped
- HARD_LIMIT_RSS_KB = ENV.fetch('SIDEKIQ_MEMORY_KILLER_HARD_LIMIT_RSS', MAX_MEMORY_KB).to_i
- # RSS in range (soft_limit_rss, hard_limit_rss) is allowed for GRACE_BALLOON_SECONDS
- GRACE_BALLOON_SECONDS = ENV.fetch('SIDEKIQ_MEMORY_KILLER_GRACE_TIME', 15 * 60).to_i
- # Check RSS every CHECK_INTERVAL_SECONDS, minimum 2 seconds
- CHECK_INTERVAL_SECONDS = [ENV.fetch('SIDEKIQ_MEMORY_KILLER_CHECK_INTERVAL', 3).to_i, 2].max
- # Give Sidekiq up to 30 seconds to allow existing jobs to finish after exceeding the limit
- SHUTDOWN_TIMEOUT_SECONDS = ENV.fetch('SIDEKIQ_MEMORY_KILLER_SHUTDOWN_WAIT', 30).to_i
- # Developer/admin should always set `memory_killer_max_memory_growth_kb` explicitly
- # In case not set, default to 300M. This is for extra-safe.
- DEFAULT_MAX_MEMORY_GROWTH_KB = 300_000
-
- # Phases of memory killer
- PHASE = {
- running: 1,
- above_soft_limit: 2,
- stop_fetching_new_jobs: 3,
- shutting_down: 4,
- killing_sidekiq: 5
- }.freeze
-
- def initialize
- super
-
- @enabled = true
- @metrics = init_metrics
- @sidekiq_daemon_monitor = Gitlab::SidekiqDaemon::Monitor.instance
- end
-
- private
-
- def init_metrics
- {
- sidekiq_current_rss: ::Gitlab::Metrics.gauge(:sidekiq_current_rss, 'Current RSS of Sidekiq Worker'),
- sidekiq_memory_killer_soft_limit_rss: ::Gitlab::Metrics.gauge(:sidekiq_memory_killer_soft_limit_rss, 'Current soft_limit_rss of Sidekiq Worker'),
- sidekiq_memory_killer_hard_limit_rss: ::Gitlab::Metrics.gauge(:sidekiq_memory_killer_hard_limit_rss, 'Current hard_limit_rss of Sidekiq Worker'),
- sidekiq_memory_killer_phase: ::Gitlab::Metrics.gauge(:sidekiq_memory_killer_phase, 'Current phase of Sidekiq Worker'),
- sidekiq_memory_killer_running_jobs: ::Gitlab::Metrics.counter(:sidekiq_memory_killer_running_jobs_total, 'Current running jobs when limit was reached')
- }
- end
-
- def refresh_state(phase)
- @phase = PHASE.fetch(phase)
- @current_rss = get_rss_kb
- @soft_limit_rss = get_soft_limit_rss_kb
- @hard_limit_rss = get_hard_limit_rss_kb
- @memory_total = get_memory_total_kb
-
- # track the current state as prometheus gauges
- @metrics[:sidekiq_memory_killer_phase].set({}, @phase)
- @metrics[:sidekiq_current_rss].set({}, @current_rss)
- @metrics[:sidekiq_memory_killer_soft_limit_rss].set({}, @soft_limit_rss)
- @metrics[:sidekiq_memory_killer_hard_limit_rss].set({}, @hard_limit_rss)
- end
-
- def run_thread
- Sidekiq.logger.info(
- class: self.class.to_s,
- action: 'start',
- pid: pid,
- message: 'Starting Gitlab::SidekiqDaemon::MemoryKiller Daemon'
- )
-
- while enabled?
- begin
- sleep(CHECK_INTERVAL_SECONDS)
- restart_sidekiq unless rss_within_range?
- rescue StandardError => e
- log_exception(e, __method__)
- rescue Exception => e # rubocop:disable Lint/RescueException
- log_exception(e, __method__)
- raise e
- end
- end
- ensure
- Sidekiq.logger.warn(
- class: self.class.to_s,
- action: 'stop',
- pid: pid,
- message: 'Stopping Gitlab::SidekiqDaemon::MemoryKiller Daemon'
- )
- end
-
- def log_exception(exception, method)
- Sidekiq.logger.warn(
- class: self.class.to_s,
- pid: pid,
- message: "Exception from #{method}: #{exception.message}"
- )
- end
-
- def stop_working
- @enabled = false
- end
-
- def enabled?
- @enabled
- end
-
- def restart_sidekiq
- return if Feature.enabled?(:sidekiq_memory_killer_read_only_mode, type: :ops)
-
- # Tell Sidekiq to stop fetching new jobs
- # We first SIGNAL and then wait given time
- # We also monitor a number of running jobs and allow to restart early
- refresh_state(:stop_fetching_new_jobs)
- signal_and_wait(SHUTDOWN_TIMEOUT_SECONDS, 'SIGTSTP', 'stop fetching new jobs')
- return unless enabled?
-
- # Tell sidekiq to restart itself
- # Keep extra safe to wait `Sidekiq[:timeout] + 2` seconds before SIGKILL
- refresh_state(:shutting_down)
- signal_and_wait(Sidekiq[:timeout] + 2, 'SIGTERM', 'gracefully shut down')
- return unless enabled?
-
- # Ideally we should never reach this condition
- # Wait for Sidekiq to shutdown gracefully, and kill it if it didn't
- # Kill the whole pgroup, so we can be sure no children are left behind
- refresh_state(:killing_sidekiq)
- signal_pgroup('SIGKILL', 'die')
- end
-
- def rss_within_range?
- refresh_state(:running)
-
- deadline = Gitlab::Metrics::System.monotonic_time + GRACE_BALLOON_SECONDS.seconds
- loop do
- return true unless enabled?
-
- # RSS go above hard limit should trigger forcible shutdown right away
- break if @current_rss > @hard_limit_rss
-
- # RSS go below the soft limit
- return true if @current_rss < @soft_limit_rss
-
- # RSS did not go below the soft limit within deadline, restart
- break if Gitlab::Metrics::System.monotonic_time > deadline
-
- sleep(CHECK_INTERVAL_SECONDS)
-
- refresh_state(:above_soft_limit)
-
- log_rss_out_of_range(false)
- end
-
- # There are two chances to break from loop:
- # - above hard limit, or
- # - above soft limit after deadline
- # When `above hard limit`, it immediately go to `stop_fetching_new_jobs`
- # So ignore `above hard limit` and always set `above_soft_limit` here
- refresh_state(:above_soft_limit)
- log_rss_out_of_range
-
- false
- end
-
- def log_rss_out_of_range(deadline_exceeded = true)
- reason = out_of_range_description(@current_rss,
- @hard_limit_rss,
- @soft_limit_rss,
- deadline_exceeded)
-
- running_jobs = fetch_running_jobs
-
- Sidekiq.logger.warn(
- class: self.class.to_s,
- pid: pid,
- message: 'Sidekiq worker RSS out of range',
- current_rss: @current_rss,
- soft_limit_rss: @soft_limit_rss,
- hard_limit_rss: @hard_limit_rss,
- memory_total_kb: @memory_total,
- reason: reason,
- running_jobs: running_jobs)
-
- increment_worker_counters(running_jobs, deadline_exceeded)
- end
-
- def increment_worker_counters(running_jobs, deadline_exceeded)
- running_jobs.each do |job|
- @metrics[:sidekiq_memory_killer_running_jobs].increment({ worker_class: job[:worker_class], deadline_exceeded: deadline_exceeded })
- end
- end
-
- def fetch_running_jobs
- @sidekiq_daemon_monitor.jobs.map do |jid, job|
- {
- jid: jid,
- worker_class: job[:worker_class].name
- }
- end
- end
-
- def out_of_range_description(rss, hard_limit, soft_limit, deadline_exceeded)
- if rss > hard_limit
- "current_rss(#{rss}) > hard_limit_rss(#{hard_limit})"
- elsif deadline_exceeded
- "current_rss(#{rss}) > soft_limit_rss(#{soft_limit}) longer than GRACE_BALLOON_SECONDS(#{GRACE_BALLOON_SECONDS})"
- else
- "current_rss(#{rss}) > soft_limit_rss(#{soft_limit})"
- end
- end
-
- def get_memory_total_kb
- Gitlab::Metrics::System.memory_total / 1.kilobytes
- end
-
- def get_rss_kb
- Gitlab::Metrics::System.memory_usage_rss[:total] / 1.kilobytes
- end
-
- def get_soft_limit_rss_kb
- SOFT_LIMIT_RSS_KB + rss_increase_by_jobs
- end
-
- def get_hard_limit_rss_kb
- HARD_LIMIT_RSS_KB
- end
-
- def signal_and_wait(time, signal, explanation)
- Sidekiq.logger.warn(
- class: self.class.to_s,
- pid: pid,
- signal: signal,
- explanation: explanation,
- wait_time: time,
- message: "Sending signal and waiting"
- )
- Process.kill(signal, pid)
-
- deadline = Gitlab::Metrics::System.monotonic_time + time
-
- # Sleep until thread killed or timeout reached
- sleep(CHECK_INTERVAL_SECONDS) while enabled? && Gitlab::Metrics::System.monotonic_time < deadline
- end
-
- def signal_pgroup(signal, explanation)
- if Process.getpgrp == pid
- pid_or_pgrp_str = 'PGRP'
- pid_to_signal = 0
- else
- pid_or_pgrp_str = 'PID'
- pid_to_signal = pid
- end
-
- Sidekiq.logger.warn(
- class: self.class.to_s,
- signal: signal,
- pid: pid,
- message: "sending Sidekiq worker #{pid_or_pgrp_str}-#{pid} #{signal} (#{explanation})"
- )
- Process.kill(signal, pid_to_signal)
- end
-
- def rss_increase_by_jobs
- @sidekiq_daemon_monitor.jobs.sum do |_, job|
- rss_increase_by_job(job)
- end
- end
-
- def rss_increase_by_job(job)
- memory_growth_kb = get_job_options(job, 'memory_killer_memory_growth_kb', 0).to_i
- max_memory_growth_kb = get_job_options(job, 'memory_killer_max_memory_growth_kb', DEFAULT_MAX_MEMORY_GROWTH_KB).to_i
-
- return 0 if memory_growth_kb == 0
-
- time_elapsed = [Gitlab::Metrics::System.monotonic_time - job[:started_at], 0].max
- [memory_growth_kb * time_elapsed, max_memory_growth_kb].min
- end
-
- def get_job_options(job, key, default)
- job[:worker_class].sidekiq_options.fetch(key, default)
- rescue StandardError
- default
- end
-
- def pid
- Process.pid
- end
- end
- end
-end
diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb
index b6e2209b475..3ed9c1743ed 100644
--- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb
+++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb
@@ -147,7 +147,10 @@ module Gitlab
end
local cookie = cmsgpack.unpack(cookie_msgpack)
cookie.deduplicated = "1"
- redis.call("set", KEYS[1], cmsgpack.pack(cookie), "ex", redis.call("ttl", KEYS[1]))
+ local ttl = redis.call("ttl", KEYS[1])
+ if ttl > 0 then
+ redis.call("set", KEYS[1], cmsgpack.pack(cookie), "ex", ttl)
+ end
LUA
def should_reschedule?
@@ -220,7 +223,12 @@ module Gitlab
end
def cookie_key
- "#{idempotency_key}:cookie:v2"
+ # This duplicates `Gitlab::Redis::Queues::SIDEKIQ_NAMESPACE` both here and in `#idempotency_key`
+ # This is because `Sidekiq.redis` used to add this prefix automatically through `redis-namespace`
+ # and we did not notice this in https://gitlab.com/gitlab-org/gitlab/-/merge_requests/25447
+ # Now we're keeping this as-is to avoid a key-migration when redis-namespace gets
+ # removed from Sidekiq: https://gitlab.com/groups/gitlab-com/gl-infra/-/epics/944
+ "#{Gitlab::Redis::Queues::SIDEKIQ_NAMESPACE}:#{idempotency_key}:cookie:v2"
end
def get_cookie
@@ -252,7 +260,7 @@ module Gitlab
end
def with_redis(&block)
- Sidekiq.redis(&block) # rubocop:disable Cop/SidekiqRedisCall
+ Gitlab::Redis::Queues.with(&block) # rubocop:disable Cop/RedisQueueUsage, CodeReuse/ActiveRecord
end
end
end
diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/deduplicates_when_scheduling.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/deduplicates_when_scheduling.rb
index 0fc95534e2a..b065190f656 100644
--- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/deduplicates_when_scheduling.rb
+++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/deduplicates_when_scheduling.rb
@@ -23,7 +23,7 @@ module Gitlab
duplicate_job.set_deduplicated_flag!(expiry)
Gitlab::SidekiqLogging::DeduplicationLogger.instance.deduplicated_log(
- job, "dropped #{strategy_name}", duplicate_job.options)
+ job, strategy_name, duplicate_job.options)
return false
end
end
diff --git a/lib/gitlab/sidekiq_middleware/server_metrics.rb b/lib/gitlab/sidekiq_middleware/server_metrics.rb
index e36f61be3b3..b3c3c94a0a3 100644
--- a/lib/gitlab/sidekiq_middleware/server_metrics.rb
+++ b/lib/gitlab/sidekiq_middleware/server_metrics.rb
@@ -17,6 +17,9 @@ module Gitlab
SIDEKIQ_JOB_DURATION_BUCKETS = [10, 300].freeze
SIDEKIQ_QUEUE_DURATION_BUCKETS = [10, 60].freeze
+ # These labels from Gitlab::SidekiqMiddleware::MetricsHelper are included in SLI metrics
+ SIDEKIQ_SLI_LABELS = [:worker, :feature_category, :urgency].freeze
+
class << self
include ::Gitlab::SidekiqMiddleware::MetricsHelper
@@ -47,17 +50,21 @@ module Gitlab
return unless ::Feature.enabled?(:sidekiq_job_completion_metric_initialize)
+ possible_sli_labels = []
::Gitlab::SidekiqConfig.current_worker_queue_mappings.each do |worker, queue|
worker_class = worker.safe_constantize
next unless worker_class
base_labels = create_labels(worker_class, queue, {})
+ possible_sli_labels << base_labels.slice(*SIDEKIQ_SLI_LABELS)
%w[done fail].each do |status|
metrics[:sidekiq_jobs_completion_seconds].get(base_labels.merge(job_status: status))
end
end
+
+ Gitlab::Metrics::SidekiqSlis.initialize_slis!(possible_sli_labels) if ::Feature.enabled?(:sidekiq_execution_application_slis)
end
end
@@ -134,6 +141,12 @@ module Gitlab
@metrics[:sidekiq_load_balancing_count].increment(labels.merge(load_balancing_labels), 1)
end
+
+ if ::Feature.enabled?(:sidekiq_execution_application_slis)
+ sli_labels = labels.slice(*SIDEKIQ_SLI_LABELS)
+ Gitlab::Metrics::SidekiqSlis.record_execution_apdex(sli_labels, monotonic_time) if job_succeeded
+ Gitlab::Metrics::SidekiqSlis.record_execution_error(sli_labels, !job_succeeded)
+ end
end
end
diff --git a/lib/gitlab/slash_commands/global_slack_handler.rb b/lib/gitlab/slash_commands/global_slack_handler.rb
new file mode 100644
index 00000000000..9bcc1f72b96
--- /dev/null
+++ b/lib/gitlab/slash_commands/global_slack_handler.rb
@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module SlashCommands
+ class GlobalSlackHandler
+ attr_reader :project_alias, :params
+
+ def initialize(params)
+ @project_alias, command = parse_command_text(params)
+ @params = params.merge(text: command, original_command: params[:text])
+ end
+
+ def trigger
+ return false unless valid_token?
+ return Gitlab::SlashCommands::ApplicationHelp.new(nil, params).execute if help_command?
+
+ unless slack_integration = find_slack_integration
+ error_message = 'GitLab error: project or alias not found'
+ return Gitlab::SlashCommands::Presenters::Error.new(error_message).message
+ end
+
+ chat_user = ChatNames::FindUserService.new(params[:team_id], params[:user_id]).execute
+ integration = slack_integration.integration
+
+ if chat_user&.user
+ Gitlab::SlashCommands::Command.new(integration.project, chat_user, params).execute
+ else
+ url = ChatNames::AuthorizeUserService.new(params).execute
+ Gitlab::SlashCommands::Presenters::Access.new(url).authorize
+ end
+ end
+
+ private
+
+ def valid_token?
+ ActiveSupport::SecurityUtils.secure_compare(
+ Gitlab::CurrentSettings.current_application_settings
+ .slack_app_verification_token,
+ params[:token]
+ )
+ end
+
+ def help_command?
+ params[:original_command] == 'help'
+ end
+
+ # rubocop: disable CodeReuse/ActiveRecord
+ def find_slack_integration
+ if project_alias.nil?
+ SlackIntegration.find_by(team_id: params[:team_id])
+ else
+ SlackIntegration.find_by(team_id: params[:team_id], alias: project_alias)
+ end
+ end
+ # rubocop: enable CodeReuse/ActiveRecord
+
+ # Splits the command
+ # '/gitlab help' => [nil, 'help']
+ # '/gitlab group/project issue new some title' => ['group/project', 'issue new some title']
+ def parse_command_text(params)
+ if params[:text] == 'incident declare'
+ [nil, params[:text]]
+ else
+ fragments = params[:text].split(/\s/, 2)
+ fragments.size == 1 ? [nil, fragments.first] : fragments
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/slash_commands/incident_management/incident_new.rb b/lib/gitlab/slash_commands/incident_management/incident_new.rb
index 722fcff151d..a43235bdeb6 100644
--- a/lib/gitlab/slash_commands/incident_management/incident_new.rb
+++ b/lib/gitlab/slash_commands/incident_management/incident_new.rb
@@ -5,11 +5,11 @@ module Gitlab
module IncidentManagement
class IncidentNew < IncidentCommand
def self.help_message
- 'incident declare'
+ 'incident declare *(Beta)*'
end
- def self.allowed?(project, user)
- Feature.enabled?(:incident_declare_slash_command, user) && can?(user, :create_incident, project)
+ def self.allowed?(_project, _user)
+ Feature.enabled?(:incident_declare_slash_command)
end
def self.match(text)
diff --git a/lib/gitlab/slug/environment.rb b/lib/gitlab/slug/environment.rb
index fd70def8e7c..2305fcd0061 100644
--- a/lib/gitlab/slug/environment.rb
+++ b/lib/gitlab/slug/environment.rb
@@ -21,7 +21,7 @@ module Gitlab
slugified = name.to_s.downcase.gsub(/[^a-z0-9]/, '-')
# Must start with a letter
- slugified = 'env-' + slugified unless slugified.match?(/^[a-z]/)
+ slugified = +"env-#{slugified}" unless slugified.match?(/^[a-z]/)
# Repeated dashes are invalid (OpenShift limitation)
slugified.squeeze!('-')
diff --git a/lib/gitlab/source.rb b/lib/gitlab/source.rb
new file mode 100644
index 00000000000..0e9fb39156d
--- /dev/null
+++ b/lib/gitlab/source.rb
@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Gitlab
+ class Source # rubocop:disable Gitlab/NamespacedClass
+ class << self
+ def ref
+ return Gitlab.revision if Gitlab.pre_release?
+
+ "v#{Gitlab::VERSION}"
+ end
+
+ def release_url
+ path = if Gitlab.pre_release?
+ url_helpers.namespace_project_commits_path(group, project, ref)
+ else
+ url_helpers.namespace_project_tag_path(group, project, ref)
+ end
+
+ Gitlab::Utils.append_path(host_url, path)
+ end
+
+ private
+
+ def host_url
+ Gitlab::Saas.com_url
+ end
+
+ def group
+ 'gitlab-org'
+ end
+
+ def project
+ 'gitlab-foss'
+ end
+
+ def url_helpers
+ Rails.application.routes.url_helpers
+ end
+ end
+ end
+end
+
+Gitlab::Source.prepend_mod
diff --git a/lib/gitlab/spamcheck/client.rb b/lib/gitlab/spamcheck/client.rb
index 0b9f3baa4de..222dd54b7b4 100644
--- a/lib/gitlab/spamcheck/client.rb
+++ b/lib/gitlab/spamcheck/client.rb
@@ -3,19 +3,13 @@ require 'spamcheck'
module Gitlab
module Spamcheck
+ Error = Class.new(StandardError)
+
class Client
include ::Spam::SpamConstants
DEFAULT_TIMEOUT_SECS = 2
- VERDICT_MAPPING = {
- ::Spamcheck::SpamVerdict::Verdict::ALLOW => ALLOW,
- ::Spamcheck::SpamVerdict::Verdict::CONDITIONAL_ALLOW => CONDITIONAL_ALLOW,
- ::Spamcheck::SpamVerdict::Verdict::DISALLOW => DISALLOW,
- ::Spamcheck::SpamVerdict::Verdict::BLOCK => BLOCK_USER,
- ::Spamcheck::SpamVerdict::Verdict::NOOP => NOOP
- }.freeze
-
ACTION_MAPPING = {
create: ::Spamcheck::Action::CREATE,
update: ::Spamcheck::Action::UPDATE
@@ -40,8 +34,9 @@ module Gitlab
pb, grpc_method = build_protobuf(**protobuf_args)
response = grpc_method.call(pb, metadata: metadata)
- verdict = convert_verdict_to_gitlab_constant(response.verdict)
- [verdict, response.extra_attributes.to_h, response.error]
+ raise Error, response.error unless response.error.blank?
+
+ Result.new(response)
end
private
@@ -53,19 +48,16 @@ module Gitlab
when Snippet
[::Spamcheck::Snippet, grpc_client.method(:check_for_spam_snippet)]
else
- raise ArgumentError, "Not a spammable type: #{spammable.class.name}"
+ [::Spamcheck::Generic, grpc_client.method(:check_for_spam_generic)]
end
end
- def convert_verdict_to_gitlab_constant(verdict)
- VERDICT_MAPPING.fetch(::Spamcheck::SpamVerdict::Verdict.resolve(verdict), verdict)
- end
-
def build_protobuf(spammable:, user:, context:, extra_features:)
protobuf_class, grpc_method = get_spammable_mappings(spammable)
pb = protobuf_class.new(**extra_features)
- pb.title = spammable.spam_title || ''
- pb.description = spammable.spam_description || ''
+ pb.title = spammable.spam_title || '' if pb.respond_to?(:title)
+ pb.description = spammable.spam_description || '' if pb.respond_to?(:description)
+ pb.text = spammable.spammable_text || '' if pb.respond_to?(:text)
pb.created_at = convert_to_pb_timestamp(spammable.created_at) if spammable.created_at
pb.updated_at = convert_to_pb_timestamp(spammable.updated_at) if spammable.updated_at
pb.action = ACTION_MAPPING.fetch(context.fetch(:action)) if context.has_key?(:action)
@@ -82,8 +74,10 @@ module Gitlab
def build_user_protobuf(user)
user_pb = ::Spamcheck::User.new
user_pb.username = user.username
+ user_pb.id = user.id
user_pb.org = user.organization || ''
user_pb.created_at = convert_to_pb_timestamp(user.created_at)
+ user_pb.abuse_metadata = Google::Protobuf::Map.new(:string, :float, user.abuse_metadata)
user_pb.emails << build_email(user.email, user.confirmed?)
diff --git a/lib/gitlab/spamcheck/result.rb b/lib/gitlab/spamcheck/result.rb
new file mode 100644
index 00000000000..7cf6b020bf5
--- /dev/null
+++ b/lib/gitlab/spamcheck/result.rb
@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Spamcheck
+ class Result
+ include ::Spam::SpamConstants
+ attr_reader :response
+
+ VERDICT_MAPPING = {
+ ::Spamcheck::SpamVerdict::Verdict::ALLOW => ALLOW,
+ ::Spamcheck::SpamVerdict::Verdict::CONDITIONAL_ALLOW => CONDITIONAL_ALLOW,
+ ::Spamcheck::SpamVerdict::Verdict::DISALLOW => DISALLOW,
+ ::Spamcheck::SpamVerdict::Verdict::BLOCK => BLOCK_USER,
+ ::Spamcheck::SpamVerdict::Verdict::NOOP => NOOP
+ }.freeze
+
+ def initialize(response)
+ @response = response
+ end
+
+ def score
+ response.score
+ end
+
+ def verdict
+ VERDICT_MAPPING.fetch(::Spamcheck::SpamVerdict::Verdict.resolve(response.verdict), ALLOW)
+ end
+
+ def evaluated?
+ response.evaluated
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/subscription_portal.rb b/lib/gitlab/subscription_portal.rb
index 7494f0584d0..1d9ecb624b2 100644
--- a/lib/gitlab/subscription_portal.rb
+++ b/lib/gitlab/subscription_portal.rb
@@ -2,22 +2,6 @@
module Gitlab
module SubscriptionPortal
- def self.default_subscriptions_url
- if ::Gitlab.dev_or_test_env?
- 'https://customers.staging.gitlab.com'
- else
- 'https://customers.gitlab.com'
- end
- end
-
- def self.subscriptions_url
- ENV.fetch('CUSTOMER_PORTAL_URL', default_subscriptions_url)
- end
-
- def self.payment_form_url
- "#{self.subscriptions_url}/payment_forms/cc_validation"
- end
-
def self.payment_validation_form_id
"payment_method_validation"
end
@@ -26,54 +10,6 @@ module Gitlab
"cc_registration_validation"
end
- def self.registration_validation_form_url
- "#{self.subscriptions_url}/payment_forms/cc_registration_validation"
- end
-
- def self.subscriptions_comparison_url
- 'https://about.gitlab.com/pricing/gitlab-com/feature-comparison'
- end
-
- def self.subscriptions_graphql_url
- "#{self.subscriptions_url}/graphql"
- end
-
- def self.subscriptions_more_minutes_url
- "#{self.subscriptions_url}/buy_pipeline_minutes"
- end
-
- def self.subscriptions_more_storage_url
- "#{self.subscriptions_url}/buy_storage"
- end
-
- def self.subscriptions_manage_url
- "#{self.subscriptions_url}/subscriptions"
- end
-
- def self.subscriptions_gitlab_plans_url
- "#{self.subscriptions_url}/gitlab_plans"
- end
-
- def self.subscriptions_instance_review_url
- "#{self.subscriptions_url}/instance_review"
- end
-
- def self.add_extra_seats_url(group_id)
- "#{self.subscriptions_url}/gitlab/namespaces/#{group_id}/extra_seats"
- end
-
- def self.upgrade_subscription_url(group_id, plan_id)
- "#{self.subscriptions_url}/gitlab/namespaces/#{group_id}/upgrade/#{plan_id}"
- end
-
- def self.renew_subscription_url(group_id)
- "#{self.subscriptions_url}/gitlab/namespaces/#{group_id}/renew"
- end
-
- def self.edit_account_url
- "#{self.subscriptions_url}/customers/edit"
- end
-
def self.subscription_portal_admin_email
ENV.fetch('SUBSCRIPTION_PORTAL_ADMIN_EMAIL', 'gl_com_api@gitlab.com')
end
@@ -89,9 +25,8 @@ module Gitlab
end
Gitlab::SubscriptionPortal.prepend_mod
-Gitlab::SubscriptionPortal::SUBSCRIPTIONS_URL = Gitlab::SubscriptionPortal.subscriptions_url.freeze
-Gitlab::SubscriptionPortal::PAYMENT_FORM_URL = Gitlab::SubscriptionPortal.payment_form_url.freeze
Gitlab::SubscriptionPortal::PAYMENT_VALIDATION_FORM_ID = Gitlab::SubscriptionPortal.payment_validation_form_id.freeze
Gitlab::SubscriptionPortal::RENEWAL_SERVICE_EMAIL = Gitlab::SubscriptionPortal.renewal_service_email.freeze
-Gitlab::SubscriptionPortal::REGISTRATION_VALIDATION_FORM_URL = Gitlab::SubscriptionPortal.registration_validation_form_url.freeze
Gitlab::SubscriptionPortal::REGISTRATION_VALIDATION_FORM_ID = Gitlab::SubscriptionPortal.registration_validation_form_id.freeze
+Gitlab::SubscriptionPortal::SUBSCRIPTION_PORTAL_ADMIN_EMAIL = Gitlab::SubscriptionPortal.subscription_portal_admin_email.freeze
+Gitlab::SubscriptionPortal::SUBSCRIPTION_PORTAL_ADMIN_TOKEN = Gitlab::SubscriptionPortal.subscription_portal_admin_token.freeze
diff --git a/lib/gitlab/task_helpers.rb b/lib/gitlab/task_helpers.rb
index 9dba8c99b99..b9800a4db73 100644
--- a/lib/gitlab/task_helpers.rb
+++ b/lib/gitlab/task_helpers.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
require 'rainbow/ext/string'
-require_dependency 'gitlab/utils/strong_memoize'
+require_relative 'utils/strong_memoize'
# rubocop:disable Rails/Output
module Gitlab
diff --git a/lib/gitlab/timeless.rb b/lib/gitlab/timeless.rb
index ed0b7b4ed87..55cd9d7e6fb 100644
--- a/lib/gitlab/timeless.rb
+++ b/lib/gitlab/timeless.rb
@@ -2,17 +2,11 @@
module Gitlab
module Timeless
- def self.timeless(model, &block)
+ def self.timeless(model)
original_record_timestamps = model.record_timestamps
model.record_timestamps = false
- # negative arity means arguments are optional
- if block.arity == 1 || block.arity < 0
- yield(model)
- else
- yield
- end
-
+ yield model
ensure
model.record_timestamps = original_record_timestamps
end
diff --git a/lib/gitlab/tracking.rb b/lib/gitlab/tracking.rb
index 45f836f10d3..52aee4d2d45 100644
--- a/lib/gitlab/tracking.rb
+++ b/lib/gitlab/tracking.rb
@@ -8,13 +8,35 @@ module Gitlab
end
def event(category, action, label: nil, property: nil, value: nil, context: [], project: nil, user: nil, namespace: nil, **extra) # rubocop:disable Metrics/ParameterLists
- contexts = [Tracking::StandardContext.new(project: project, user: user, namespace: namespace, **extra).to_context, *context]
+ action = action.to_s
+
+ project_id = project.is_a?(Integer) ? project : project&.id
+
+ contexts = [
+ Tracking::StandardContext.new(
+ namespace_id: namespace&.id,
+ plan_name: namespace&.actual_plan_name,
+ project_id: project_id,
+ user_id: user&.id,
+ **extra).to_context, *context
+ ]
+ track_struct_event(tracker, category, action, label: label, property: property, value: value, contexts: contexts)
+ end
+
+ def database_event(category, action, label: nil, property: nil, value: nil, context: [], project: nil, user: nil, namespace: nil, **extra) # rubocop:disable Metrics/ParameterLists
action = action.to_s
+ destination = Gitlab::Tracking::Destinations::DatabaseEventsSnowplow.new
+ contexts = [
+ Tracking::StandardContext.new(
+ namespace_id: namespace&.id,
+ plan_name: namespace&.actual_plan_name,
+ project_id: project&.id,
+ user_id: user&.id,
+ **extra).to_context, *context
+ ]
- tracker.event(category, action, label: label, property: property, value: value, context: contexts)
- rescue StandardError => error
- Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error, snowplow_category: category, snowplow_action: action)
+ track_struct_event(destination, category, action, label: label, property: property, value: value, contexts: contexts)
end
def definition(basename, category: nil, action: nil, label: nil, property: nil, value: nil, context: [], project: nil, user: nil, namespace: nil, **extra) # rubocop:disable Metrics/ParameterLists
@@ -42,12 +64,19 @@ module Gitlab
def snowplow_micro_enabled?
Rails.env.development? && Gitlab.config.snowplow_micro.enabled
- rescue Settingslogic::MissingSetting
+ rescue GitlabSettings::MissingSetting
false
end
private
+ def track_struct_event(destination, category, action, label:, property:, value:, contexts:) # rubocop:disable Metrics/ParameterLists
+ destination
+ .event(category, action, label: label, property: property, value: value, context: contexts)
+ rescue StandardError => error
+ Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error, snowplow_category: category, snowplow_action: action)
+ end
+
def tracker
@tracker ||= if snowplow_micro_enabled?
Gitlab::Tracking::Destinations::SnowplowMicro.new
diff --git a/lib/gitlab/tracking/destinations/database_events_snowplow.rb b/lib/gitlab/tracking/destinations/database_events_snowplow.rb
new file mode 100644
index 00000000000..e3512bc4916
--- /dev/null
+++ b/lib/gitlab/tracking/destinations/database_events_snowplow.rb
@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Tracking
+ module Destinations
+ class DatabaseEventsSnowplow < Snowplow
+ extend ::Gitlab::Utils::Override
+
+ HOSTNAME = 'db-snowplow.trx.gitlab.net'
+
+ override :enabled?
+ # database events are only collected for SaaS instance
+ def enabled?
+ ::Gitlab.dev_or_test_env? || ::Gitlab.com?
+ end
+
+ override :hostname
+ def hostname
+ return HOSTNAME if ::Gitlab.com?
+
+ 'localhost:9091'
+ end
+
+ private
+
+ override :increment_failed_events_emissions
+ def increment_failed_events_emissions(value)
+ Gitlab::Metrics.counter(
+ :gitlab_db_events_snowplow_failed_events_total,
+ 'Number of failed Snowplow events emissions'
+ ).increment({}, value.to_i)
+ end
+
+ override :increment_successful_events_emissions
+ def increment_successful_events_emissions(value)
+ Gitlab::Metrics.counter(
+ :gitlab_db_events_snowplow_successful_events_total,
+ 'Number of successful Snowplow events emissions'
+ ).increment({}, value.to_i)
+ end
+
+ override :increment_total_events_counter
+ def increment_total_events_counter
+ Gitlab::Metrics.counter(
+ :gitlab_db_events_snowplow_events_total,
+ 'Number of Snowplow events'
+ ).increment
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/tracking/destinations/snowplow_micro.rb b/lib/gitlab/tracking/destinations/snowplow_micro.rb
index 09480f26106..e15c03b6808 100644
--- a/lib/gitlab/tracking/destinations/snowplow_micro.rb
+++ b/lib/gitlab/tracking/destinations/snowplow_micro.rb
@@ -53,7 +53,7 @@ module Gitlab
url = Gitlab.config.snowplow_micro.address
scheme = Gitlab.config.gitlab.https ? 'https' : 'http'
"#{scheme}://#{url}"
- rescue Settingslogic::MissingSetting
+ rescue GitlabSettings::MissingSetting
DEFAULT_URI
end
end
diff --git a/lib/gitlab/tracking/standard_context.rb b/lib/gitlab/tracking/standard_context.rb
index 50467de44b8..62c45368410 100644
--- a/lib/gitlab/tracking/standard_context.rb
+++ b/lib/gitlab/tracking/standard_context.rb
@@ -6,15 +6,16 @@ module Gitlab
GITLAB_STANDARD_SCHEMA_URL = 'iglu:com.gitlab/gitlab_standard/jsonschema/1-0-8'
GITLAB_RAILS_SOURCE = 'gitlab-rails'
- def initialize(namespace: nil, project: nil, user: nil, **extra)
- check_argument_type(:namespace, namespace, [Namespace])
- check_argument_type(:project, project, [Project, Integer])
- check_argument_type(:user, user, [User, DeployToken])
-
- @namespace = namespace
- @plan = namespace&.actual_plan_name
- @project = project
- @user = user
+ def initialize(namespace_id: nil, plan_name: nil, project_id: nil, user_id: nil, **extra)
+ check_argument_type(:namespace_id, namespace_id, [Integer])
+ check_argument_type(:plan_name, plan_name, [String])
+ check_argument_type(:project_id, project_id, [Integer])
+ check_argument_type(:user_id, user_id, [Integer])
+
+ @namespace_id = namespace_id
+ @plan_name = plan_name
+ @project_id = project_id
+ @user_id = user_id
@extra = extra
end
@@ -40,25 +41,21 @@ module Gitlab
private
- attr_accessor :namespace, :project, :extra, :plan, :user
+ attr_accessor :namespace_id, :project_id, :extra, :plan_name, :user_id
def to_h
{
environment: environment,
source: source,
- plan: plan,
+ plan: plan_name,
extra: extra,
- user_id: user&.id,
- namespace_id: namespace&.id,
+ user_id: user_id,
+ namespace_id: namespace_id,
project_id: project_id,
context_generated_at: Time.current
}
end
- def project_id
- project.is_a?(Integer) ? project : project&.id
- end
-
def check_argument_type(argument_name, argument_value, allowed_classes)
return if argument_value.nil? || allowed_classes.any? { |allowed_class| argument_value.is_a?(allowed_class) }
diff --git a/lib/gitlab/untrusted_regexp.rb b/lib/gitlab/untrusted_regexp.rb
index 7c7bda3a8f9..fe3377dae68 100644
--- a/lib/gitlab/untrusted_regexp.rb
+++ b/lib/gitlab/untrusted_regexp.rb
@@ -10,6 +10,9 @@ module Gitlab
# Not all regular expression features are available in untrusted regexes, and
# there is a strict limit on total execution time. See the RE2 documentation
# at https://github.com/google/re2/wiki/Syntax for more details.
+ #
+ # This class doesn't change any instance variables, which allows it to be frozen
+ # and setup in constants.
class UntrustedRegexp
require_dependency 're2'
@@ -21,6 +24,7 @@ module Gitlab
end
@regexp = RE2::Regexp.new(pattern, log_errors: false)
+ @scan_regexp = initialize_scan_regexp
raise RegexpError, regexp.error unless regexp.ok?
end
@@ -29,6 +33,27 @@ module Gitlab
RE2.GlobalReplace(text, regexp, rewrite)
end
+ # There is no built-in replace with block support (like `gsub`). We can accomplish
+ # the same thing by parsing and rebuilding the string with the substitutions.
+ def replace_gsub(text)
+ new_text = +''
+ remainder = text
+
+ matched = match(remainder)
+
+ until matched.nil? || matched.to_a.compact.empty?
+ partitioned = remainder.partition(matched.to_s)
+ new_text << partitioned.first
+ remainder = partitioned.last
+
+ new_text << yield(matched)
+
+ matched = match(remainder)
+ end
+
+ new_text << remainder
+ end
+
def scan(text)
matches = scan_regexp.scan(text).to_a
matches.map!(&:first) if regexp.number_of_capturing_groups == 0
@@ -87,17 +112,16 @@ module Gitlab
private
- attr_reader :regexp
+ attr_reader :regexp, :scan_regexp
# RE2 scan operates differently to Ruby scan when there are no capture
# groups, so work around it
- def scan_regexp
- @scan_regexp ||=
- if regexp.number_of_capturing_groups == 0
- RE2::Regexp.new('(' + regexp.source + ')')
- else
- regexp
- end
+ def initialize_scan_regexp
+ if regexp.number_of_capturing_groups == 0
+ RE2::Regexp.new('(' + regexp.source + ')')
+ else
+ regexp
+ end
end
end
end
diff --git a/lib/gitlab/url_blocker.rb b/lib/gitlab/url_blocker.rb
index 00e609511f2..ba50a42cd37 100644
--- a/lib/gitlab/url_blocker.rb
+++ b/lib/gitlab/url_blocker.rb
@@ -7,20 +7,40 @@ module Gitlab
class UrlBlocker
BlockedUrlError = Class.new(StandardError)
+ DENY_ALL_REQUESTS_EXCEPT_ALLOWED_DEFAULT = proc { deny_all_requests_except_allowed_app_setting }.freeze
+
+ # Result stores the validation result:
+ # uri - The original URI requested
+ # hostname - The hostname that should be used to connect. For DNS
+ # rebinding protection, this will be the resolved IP address of
+ # the hostname.
+ # use_proxy -
+ # If true, this means that the proxy server specified in the
+ # http_proxy/https_proxy environment variables should be used.
+ #
+ # If false, this either means that no proxy server was specified
+ # or that the hostname in the URL is exempt via the no_proxy
+ # environment variable. This allows the caller to disable usage
+ # of a proxy since the IP address may be used to
+ # connect. Otherwise, Net::HTTP may erroneously compare the IP
+ # address against the no_proxy list.
+ Result = Struct.new(:uri, :hostname, :use_proxy)
+
class << self
# Validates the given url according to the constraints specified by arguments.
#
- # ports - Raises error if the given URL port does is not between given ports.
+ # ports - Raises error if the given URL port is not between given ports.
# allow_localhost - Raises error if URL resolves to a localhost IP address and argument is false.
# allow_local_network - Raises error if URL resolves to a link-local address and argument is false.
# allow_object_storage - Avoid raising an error if URL resolves to an object storage endpoint and argument is true.
# ascii_only - Raises error if URL has unicode characters and argument is true.
# enforce_user - Raises error if URL user doesn't start with alphanumeric characters and argument is true.
# enforce_sanitization - Raises error if URL includes any HTML/CSS/JS tags and argument is true.
+ # deny_all_requests_except_allowed - Raises error if URL is not in the allow list and argument is true. Can be Boolean or Proc. Defaults to instance app setting.
#
- # Returns an array with [<uri>, <original-hostname>].
+ # Returns a Result object.
# rubocop:disable Metrics/ParameterLists
- def validate!(
+ def validate_url_with_proxy!(
url,
schemes:,
ports: [],
@@ -30,10 +50,11 @@ module Gitlab
ascii_only: false,
enforce_user: false,
enforce_sanitization: false,
+ deny_all_requests_except_allowed: DENY_ALL_REQUESTS_EXCEPT_ALLOWED_DEFAULT,
dns_rebind_protection: true)
# rubocop:enable Metrics/ParameterLists
- return [nil, nil] if url.nil?
+ return Result.new(nil, nil, true) if url.nil?
raise ArgumentError, 'The schemes is a required argument' if schemes.blank?
@@ -49,21 +70,35 @@ module Gitlab
ascii_only: ascii_only
)
- address_info = get_address_info(uri, dns_rebind_protection)
- return [uri, nil] unless address_info
+ begin
+ address_info = get_address_info(uri)
+ rescue SocketError
+ proxy_in_use = uri_under_proxy_setting?(uri, nil)
+
+ return Result.new(uri, nil, proxy_in_use) unless enforce_address_info_retrievable?(uri, dns_rebind_protection, deny_all_requests_except_allowed)
+
+ raise BlockedUrlError, 'Host cannot be resolved or invalid'
+ end
ip_address = ip_address(address_info)
- return [uri, nil] if domain_allowed?(uri)
+ proxy_in_use = uri_under_proxy_setting?(uri, ip_address)
+
+ # Ignore DNS rebind protection when a proxy is being used, as DNS
+ # rebinding is expected behavior.
+ dns_rebind_protection &&= !proxy_in_use
+ return Result.new(uri, nil, proxy_in_use) if domain_in_allow_list?(uri)
- protected_uri_with_hostname = enforce_uri_hostname(ip_address, uri, dns_rebind_protection)
+ protected_uri_with_hostname = enforce_uri_hostname(ip_address, uri, dns_rebind_protection, proxy_in_use)
- return protected_uri_with_hostname if ip_allowed?(ip_address, port: get_port(uri))
+ return protected_uri_with_hostname if ip_in_allow_list?(ip_address, port: get_port(uri))
# Allow url from the GitLab instance itself but only for the configured hostname and ports
return protected_uri_with_hostname if internal?(uri)
return protected_uri_with_hostname if allow_object_storage && object_storage_endpoint?(uri)
+ validate_deny_all_requests_except_allowed!(deny_all_requests_except_allowed)
+
validate_local_request(
address_info: address_info,
allow_localhost: allow_localhost,
@@ -81,6 +116,13 @@ module Gitlab
true
end
+ # For backwards compatibility, Returns an array with [<uri>, <original-hostname>].
+ # Issue for refactoring: https://gitlab.com/gitlab-org/gitlab/-/issues/410890
+ def validate!(...)
+ result = validate_url_with_proxy!(...)
+ [result.uri, result.hostname]
+ end
+
private
# Returns the given URI with IP address as hostname and the original hostname respectively
@@ -91,12 +133,12 @@ module Gitlab
#
# The original hostname is used to validate the SSL, given in that scenario
# we'll be making the request to the IP address, instead of using the hostname.
- def enforce_uri_hostname(ip_address, uri, dns_rebind_protection)
- return [uri, nil] unless dns_rebind_protection && ip_address && ip_address != uri.hostname
+ def enforce_uri_hostname(ip_address, uri, dns_rebind_protection, proxy_in_use)
+ return Result.new(uri, nil, proxy_in_use) unless dns_rebind_protection && ip_address && ip_address != uri.hostname
new_uri = uri.dup
new_uri.hostname = ip_address
- [new_uri, uri.hostname]
+ Result.new(new_uri, uri.hostname, proxy_in_use)
end
def ip_address(address_info)
@@ -115,29 +157,56 @@ module Gitlab
validate_unicode_restriction(uri) if ascii_only
end
- def get_address_info(uri, dns_rebind_protection)
+ def uri_under_proxy_setting?(uri, ip_address)
+ return false unless Gitlab.http_proxy_env?
+ # `no_proxy|NO_PROXY` specifies addresses for which the proxy is not
+ # used. If it's empty, there are no exceptions and this URI
+ # will be under proxy settings.
+ return true if no_proxy_env.blank?
+
+ # `no_proxy|NO_PROXY` is being used. We must check whether it
+ # applies to this specific URI.
+ ::URI::Generic.use_proxy?(uri.hostname, ip_address, get_port(uri), no_proxy_env)
+ end
+
+ # Returns addrinfo object for the URI.
+ #
+ # @param uri [Addressable::URI]
+ #
+ # @raise [Gitlab::UrlBlocker::BlockedUrlError, ArgumentError] - BlockedUrlError raised if host is too long.
+ #
+ # @return [Array<Addrinfo>]
+ def get_address_info(uri)
Addrinfo.getaddrinfo(uri.hostname, get_port(uri), nil, :STREAM).map do |addr|
addr.ipv6_v4mapped? ? addr.ipv6_to_ipv4 : addr
end
- rescue SocketError
- # If the dns rebinding protection is not enabled or the domain
- # is allowed we avoid the dns rebinding checks
- return if domain_allowed?(uri) || !dns_rebind_protection
+ rescue ArgumentError => error
+ # Addrinfo.getaddrinfo errors if the domain exceeds 1024 characters.
+ raise unless error.message.include?('hostname too long')
+
+ raise BlockedUrlError, "Host is too long (maximum is 1024 characters)"
+ end
+
+ def enforce_address_info_retrievable?(uri, dns_rebind_protection, deny_all_requests_except_allowed)
+ # Do not enforce if URI is in the allow list
+ return false if domain_in_allow_list?(uri)
+
+ # Enforce if the instance should block requests
+ return true if deny_all_requests_except_allowed?(deny_all_requests_except_allowed)
+
+ # Do not enforce if DNS rebinding protection is disabled
+ return false unless dns_rebind_protection
+
+ # Do not enforce if proxy is used
+ return false if Gitlab.http_proxy_env?
# In the test suite we use a lot of mocked urls that are either invalid or
# don't exist. In order to avoid modifying a ton of tests and factories
# we allow invalid urls unless the environment variable RSPEC_ALLOW_INVALID_URLS
# is not true
- return if Rails.env.test? && ENV['RSPEC_ALLOW_INVALID_URLS'] == 'true'
-
- # If the addr can't be resolved or the url is invalid (i.e http://1.1.1.1.1)
- # we block the url
- raise BlockedUrlError, "Host cannot be resolved or invalid"
- rescue ArgumentError => error
- # Addrinfo.getaddrinfo errors if the domain exceeds 1024 characters.
- raise unless error.message.include?('hostname too long')
+ return false if Rails.env.test? && ENV['RSPEC_ALLOW_INVALID_URLS'] == 'true'
- raise BlockedUrlError, "Host is too long (maximum is 1024 characters)"
+ true
end
def validate_local_request(
@@ -260,6 +329,15 @@ module Gitlab
raise BlockedUrlError, "Requests to the link local network are not allowed"
end
+ # Raises a BlockedUrlError if the instance is configured to deny all requests.
+ #
+ # This should only be called after allow list checks have been made.
+ def validate_deny_all_requests_except_allowed!(should_deny)
+ return unless deny_all_requests_except_allowed?(should_deny)
+
+ raise BlockedUrlError, "Requests to hosts and IP addresses not on the Allow List are denied"
+ end
+
# Raises a BlockedUrlError if any IP in `addrs_info` is the limited
# broadcast address.
# https://datatracker.ietf.org/doc/html/rfc919#section-7
@@ -293,8 +371,7 @@ module Gitlab
next unless section_setting && section_setting['enabled']
- # Use #to_h to avoid Settingslogic bug: https://gitlab.com/gitlab-org/gitlab/-/issues/286873
- object_store_setting = section_setting['object_store']&.to_h
+ object_store_setting = section_setting['object_store']
next unless object_store_setting && object_store_setting['enabled']
@@ -302,6 +379,15 @@ module Gitlab
end.compact.uniq
end
+ def deny_all_requests_except_allowed?(should_deny)
+ should_deny.is_a?(Proc) ? should_deny.call : should_deny
+ end
+
+ def deny_all_requests_except_allowed_app_setting
+ Gitlab::CurrentSettings.current_application_settings? &&
+ Gitlab::CurrentSettings.deny_all_requests_except_allowed?
+ end
+
def object_storage_endpoint?(uri)
enabled_object_storage_endpoints.any? do |endpoint|
endpoint_uri = URI(endpoint)
@@ -312,17 +398,21 @@ module Gitlab
end
end
- def domain_allowed?(uri)
+ def domain_in_allow_list?(uri)
Gitlab::UrlBlockers::UrlAllowlist.domain_allowed?(uri.normalized_host, port: get_port(uri))
end
- def ip_allowed?(ip_address, port: nil)
+ def ip_in_allow_list?(ip_address, port: nil)
Gitlab::UrlBlockers::UrlAllowlist.ip_allowed?(ip_address, port: port)
end
def config
Gitlab.config
end
+
+ def no_proxy_env
+ ENV['no_proxy'] || ENV['NO_PROXY']
+ end
end
end
end
diff --git a/lib/gitlab/url_blockers/ip_allowlist_entry.rb b/lib/gitlab/url_blockers/ip_allowlist_entry.rb
index b293afe166c..ff4eb86ec41 100644
--- a/lib/gitlab/url_blockers/ip_allowlist_entry.rb
+++ b/lib/gitlab/url_blockers/ip_allowlist_entry.rb
@@ -12,11 +12,32 @@ module Gitlab
end
def match?(requested_ip, requested_port = nil)
- return false unless ip.include?(requested_ip)
+ requested_ip = IPAddr.new(requested_ip) if requested_ip.is_a?(String)
+
+ return false unless ip_include?(requested_ip)
return true if port.nil?
port == requested_port
end
+
+ private
+
+ # Prior to ipaddr v1.2.3, if the allow list were the IPv4 to IPv6
+ # mapped address ::ffff:169.254.168.100 and the requested IP were
+ # 169.254.168.100 or ::ffff:169.254.168.100, the IP would be
+ # considered in the allow list. However, with
+ # https://github.com/ruby/ipaddr/pull/31, IPAddr#include? will
+ # only match if the IP versions are the same. This method
+ # preserves backwards compatibility if the versions differ by
+ # checking inclusion by coercing an IPv4 address to its IPv6
+ # mapped address.
+ def ip_include?(requested_ip)
+ return true if ip.include?(requested_ip)
+ return ip.include?(requested_ip.ipv4_mapped) if requested_ip.ipv4? && ip.ipv6?
+ return ip.ipv4_mapped.include?(requested_ip) if requested_ip.ipv6? && ip.ipv4?
+
+ false
+ end
end
end
end
diff --git a/lib/gitlab/usage/metrics/aggregates/aggregate.rb b/lib/gitlab/usage/metrics/aggregates/aggregate.rb
index b68e1ace658..a0a58534661 100644
--- a/lib/gitlab/usage/metrics/aggregates/aggregate.rb
+++ b/lib/gitlab/usage/metrics/aggregates/aggregate.rb
@@ -7,11 +7,6 @@ module Gitlab
class Aggregate
include Gitlab::Usage::TimeFrame
- # TODO: define this missing event https://gitlab.com/gitlab-org/gitlab/-/issues/385080
- EVENTS_NOT_DEFINED_YET = %w[
- i_code_review_merge_request_widget_license_compliance_warning
- ].freeze
-
def initialize(recorded_at)
@recorded_at = recorded_at
end
@@ -84,7 +79,7 @@ module Gitlab
return events if source != ::Gitlab::Usage::Metrics::Aggregates::REDIS_SOURCE
events.select do |event|
- ::Gitlab::UsageDataCounters::HLLRedisCounter.known_event?(event) || EVENTS_NOT_DEFINED_YET.include?(event)
+ ::Gitlab::UsageDataCounters::HLLRedisCounter.known_event?(event)
end
end
end
diff --git a/lib/gitlab/usage/metrics/instrumentations/count_bulk_imports_entities_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_bulk_imports_entities_metric.rb
index 642b67a3b02..ca122ccf6f3 100644
--- a/lib/gitlab/usage/metrics/instrumentations/count_bulk_imports_entities_metric.rb
+++ b/lib/gitlab/usage/metrics/instrumentations/count_bulk_imports_entities_metric.rb
@@ -23,6 +23,7 @@ module Gitlab
scope = super
scope = scope.where(source_type: source_type) if source_type.present?
scope = scope.where(status: status) if status.present?
+ scope = scope.where(has_failures: failures) if failures.present?
scope
end
@@ -34,6 +35,10 @@ module Gitlab
options[:status]
end
+ def failures
+ options[:has_failures].to_s
+ end
+
def allowed_source_types
BulkImports::Entity.source_types.keys.map(&:to_s)
end
diff --git a/lib/gitlab/usage/metrics/instrumentations/count_ci_runners_group_type_active_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_ci_runners_group_type_active_metric.rb
new file mode 100644
index 00000000000..fbf4e0f904b
--- /dev/null
+++ b/lib/gitlab/usage/metrics/instrumentations/count_ci_runners_group_type_active_metric.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Usage
+ module Metrics
+ module Instrumentations
+ class CountCiRunnersGroupTypeActiveMetric < DatabaseMetric
+ operation :count
+
+ relation { ::Ci::Runner.group_type.active }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/usage/metrics/instrumentations/count_ci_runners_group_type_active_online_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_ci_runners_group_type_active_online_metric.rb
new file mode 100644
index 00000000000..acb6de53d14
--- /dev/null
+++ b/lib/gitlab/usage/metrics/instrumentations/count_ci_runners_group_type_active_online_metric.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Usage
+ module Metrics
+ module Instrumentations
+ class CountCiRunnersGroupTypeActiveOnlineMetric < DatabaseMetric
+ operation :count
+
+ relation { ::Ci::Runner.group_type.active.online }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/usage/metrics/instrumentations/count_ci_runners_instance_type_active_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_ci_runners_instance_type_active_metric.rb
new file mode 100644
index 00000000000..d9a785679d7
--- /dev/null
+++ b/lib/gitlab/usage/metrics/instrumentations/count_ci_runners_instance_type_active_metric.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Usage
+ module Metrics
+ module Instrumentations
+ class CountCiRunnersInstanceTypeActiveMetric < DatabaseMetric
+ operation :count
+
+ relation do
+ ::Ci::Runner.instance_type.active
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/usage/metrics/instrumentations/count_ci_runners_instance_type_active_online_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_ci_runners_instance_type_active_online_metric.rb
new file mode 100644
index 00000000000..05a9c47c016
--- /dev/null
+++ b/lib/gitlab/usage/metrics/instrumentations/count_ci_runners_instance_type_active_online_metric.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Usage
+ module Metrics
+ module Instrumentations
+ class CountCiRunnersInstanceTypeActiveOnlineMetric < DatabaseMetric
+ operation :count
+
+ relation { ::Ci::Runner.instance_type.active.online }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/usage/metrics/instrumentations/count_ci_runners_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_ci_runners_metric.rb
new file mode 100644
index 00000000000..8be4955e28d
--- /dev/null
+++ b/lib/gitlab/usage/metrics/instrumentations/count_ci_runners_metric.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Usage
+ module Metrics
+ module Instrumentations
+ class CountCiRunnersMetric < DatabaseMetric
+ operation :count
+
+ relation { ::Ci::Runner }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/usage/metrics/instrumentations/count_ci_runners_project_type_active_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_ci_runners_project_type_active_metric.rb
new file mode 100644
index 00000000000..e713e85b270
--- /dev/null
+++ b/lib/gitlab/usage/metrics/instrumentations/count_ci_runners_project_type_active_metric.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Usage
+ module Metrics
+ module Instrumentations
+ class CountCiRunnersProjectTypeActiveMetric < DatabaseMetric
+ operation :count
+
+ relation { ::Ci::Runner.project_type.active }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/usage/metrics/instrumentations/count_ci_runners_project_type_active_online_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_ci_runners_project_type_active_online_metric.rb
new file mode 100644
index 00000000000..91e7c6063b8
--- /dev/null
+++ b/lib/gitlab/usage/metrics/instrumentations/count_ci_runners_project_type_active_online_metric.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Usage
+ module Metrics
+ module Instrumentations
+ class CountCiRunnersProjectTypeActiveOnlineMetric < DatabaseMetric
+ operation :count
+
+ relation { ::Ci::Runner.project_type.active.online }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/usage/metrics/instrumentations/database_mode.rb b/lib/gitlab/usage/metrics/instrumentations/database_mode.rb
new file mode 100644
index 00000000000..1b97ef4a1d2
--- /dev/null
+++ b/lib/gitlab/usage/metrics/instrumentations/database_mode.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Usage
+ module Metrics
+ module Instrumentations
+ class DatabaseMode < GenericMetric
+ value do
+ Gitlab::Database.database_mode
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/usage/metrics/instrumentations/edition_metric.rb b/lib/gitlab/usage/metrics/instrumentations/edition_metric.rb
new file mode 100644
index 00000000000..83153242703
--- /dev/null
+++ b/lib/gitlab/usage/metrics/instrumentations/edition_metric.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Usage
+ module Metrics
+ module Instrumentations
+ class EditionMetric < GenericMetric
+ value do
+ if Gitlab.ee?
+ ::License.current&.edition || 'EE Free'
+ else
+ 'CE'
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/usage/metrics/instrumentations/gitlab_dedicated_metric.rb b/lib/gitlab/usage/metrics/instrumentations/gitlab_dedicated_metric.rb
new file mode 100644
index 00000000000..b7ca5fadd5b
--- /dev/null
+++ b/lib/gitlab/usage/metrics/instrumentations/gitlab_dedicated_metric.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Usage
+ module Metrics
+ module Instrumentations
+ class GitlabDedicatedMetric < GenericMetric
+ value do
+ Gitlab::CurrentSettings.gitlab_dedicated_instance
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/usage/metrics/instrumentations/incoming_email_encrypted_secrets_enabled_metric.rb b/lib/gitlab/usage/metrics/instrumentations/incoming_email_encrypted_secrets_enabled_metric.rb
index ab9c6f87023..be3b3b3d682 100644
--- a/lib/gitlab/usage/metrics/instrumentations/incoming_email_encrypted_secrets_enabled_metric.rb
+++ b/lib/gitlab/usage/metrics/instrumentations/incoming_email_encrypted_secrets_enabled_metric.rb
@@ -6,7 +6,7 @@ module Gitlab
module Instrumentations
class IncomingEmailEncryptedSecretsEnabledMetric < GenericMetric
value do
- Gitlab::IncomingEmail.encrypted_secrets.active?
+ Gitlab::Email::IncomingEmail.encrypted_secrets.active?
end
end
end
diff --git a/lib/gitlab/usage/metrics/instrumentations/index_inconsistencies_metric.rb b/lib/gitlab/usage/metrics/instrumentations/index_inconsistencies_metric.rb
new file mode 100644
index 00000000000..409027925d1
--- /dev/null
+++ b/lib/gitlab/usage/metrics/instrumentations/index_inconsistencies_metric.rb
@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Usage
+ module Metrics
+ module Instrumentations
+ class IndexInconsistenciesMetric < GenericMetric
+ value do
+ runner = Gitlab::Database::SchemaValidation::Runner.new(structure_sql, database, validators: validators)
+
+ inconsistencies = runner.execute
+
+ inconsistencies.map do |inconsistency|
+ {
+ object_name: inconsistency.object_name,
+ inconsistency_type: inconsistency.type
+ }
+ end
+ end
+
+ class << self
+ private
+
+ def database
+ database_model = Gitlab::Database.database_base_models[Gitlab::Database::MAIN_DATABASE_NAME]
+ Gitlab::Database::SchemaValidation::Database.new(database_model.connection)
+ end
+
+ def structure_sql
+ stucture_sql_path = Rails.root.join('db/structure.sql')
+ Gitlab::Database::SchemaValidation::StructureSql.new(stucture_sql_path)
+ end
+
+ def validators
+ [
+ Gitlab::Database::SchemaValidation::Validators::MissingIndexes,
+ Gitlab::Database::SchemaValidation::Validators::DifferentDefinitionIndexes,
+ Gitlab::Database::SchemaValidation::Validators::ExtraIndexes
+ ]
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/usage/metrics/instrumentations/installation_creation_date_approximation_metric.rb b/lib/gitlab/usage/metrics/instrumentations/installation_creation_date_approximation_metric.rb
new file mode 100644
index 00000000000..e8ae4b4f906
--- /dev/null
+++ b/lib/gitlab/usage/metrics/instrumentations/installation_creation_date_approximation_metric.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Usage
+ module Metrics
+ module Instrumentations
+ class InstallationCreationDateApproximationMetric < GenericMetric
+ value do
+ [User.first, ApplicationSetting.first].compact.pluck(:created_at).compact.min
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/usage/metrics/instrumentations/installation_creation_date_metric.rb b/lib/gitlab/usage/metrics/instrumentations/installation_creation_date_metric.rb
new file mode 100644
index 00000000000..c2ca62f9eba
--- /dev/null
+++ b/lib/gitlab/usage/metrics/instrumentations/installation_creation_date_metric.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Usage
+ module Metrics
+ module Instrumentations
+ class InstallationCreationDateMetric < GenericMetric
+ value do
+ User.where(id: 1).pick(:created_at)
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/usage/metrics/instrumentations/installation_type_metric.rb b/lib/gitlab/usage/metrics/instrumentations/installation_type_metric.rb
new file mode 100644
index 00000000000..7147fc2e624
--- /dev/null
+++ b/lib/gitlab/usage/metrics/instrumentations/installation_type_metric.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Usage
+ module Metrics
+ module Instrumentations
+ class InstallationTypeMetric < GenericMetric
+ value do
+ if Rails.env.production?
+ Gitlab::INSTALLATION_TYPE
+ else
+ "gitlab-development-kit"
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/usage/metrics/instrumentations/service_desk_email_encrypted_secrets_enabled_metric.rb b/lib/gitlab/usage/metrics/instrumentations/service_desk_email_encrypted_secrets_enabled_metric.rb
index 4332043de8a..5e38339801b 100644
--- a/lib/gitlab/usage/metrics/instrumentations/service_desk_email_encrypted_secrets_enabled_metric.rb
+++ b/lib/gitlab/usage/metrics/instrumentations/service_desk_email_encrypted_secrets_enabled_metric.rb
@@ -6,7 +6,7 @@ module Gitlab
module Instrumentations
class ServiceDeskEmailEncryptedSecretsEnabledMetric < GenericMetric
value do
- Gitlab::ServiceDeskEmail.encrypted_secrets.active?
+ Gitlab::Email::ServiceDeskEmail.encrypted_secrets.active?
end
end
end
diff --git a/lib/gitlab/usage/metrics/instrumentations/version_metric.rb b/lib/gitlab/usage/metrics/instrumentations/version_metric.rb
new file mode 100644
index 00000000000..cc26268067f
--- /dev/null
+++ b/lib/gitlab/usage/metrics/instrumentations/version_metric.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Usage
+ module Metrics
+ module Instrumentations
+ class VersionMetric < GenericMetric
+ value do
+ Gitlab::VERSION
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/usage/service_ping_report.rb b/lib/gitlab/usage/service_ping_report.rb
index 1eda72ba570..3bc941260d6 100644
--- a/lib/gitlab/usage/service_ping_report.rb
+++ b/lib/gitlab/usage/service_ping_report.rb
@@ -9,7 +9,9 @@ module Gitlab
def for(output:, cached: false)
case output.to_sym
when :all_metrics_values
- with_instrumentation_classes(all_metrics_values(cached), :with_value)
+ Rails.cache.fetch(CACHE_KEY, force: !cached, expires_in: 2.weeks) do
+ with_instrumentation_classes(Gitlab::UsageData.data, :with_value)
+ end
when :metrics_queries
with_instrumentation_classes(metrics_queries, :with_instrumentation)
when :non_sql_metrics_values
@@ -27,12 +29,6 @@ module Gitlab
old_payload.with_indifferent_access.deep_merge(instrumented_payload)
end
- def all_metrics_values(cached)
- Rails.cache.fetch(CACHE_KEY, force: !cached, expires_in: 2.weeks) do
- Gitlab::UsageData.data
- end
- end
-
def metrics_queries
Gitlab::UsageDataQueries.data
end
diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb
index 53794854bd0..846bb934a3d 100644
--- a/lib/gitlab/usage_data.rb
+++ b/lib/gitlab/usage_data.rb
@@ -47,15 +47,7 @@ module Gitlab
end
def license_usage_data
- {
- recorded_at: recorded_at,
- uuid: add_metric('UuidMetric'),
- hostname: add_metric('HostnameMetric'),
- version: alt_usage_data { Gitlab::VERSION },
- installation_type: alt_usage_data { installation_type },
- active_user_count: add_metric('ActiveUserCountMetric'),
- edition: 'CE'
- }
+ { recorded_at: recorded_at }
end
def recorded_at
@@ -145,7 +137,6 @@ module Gitlab
merge_requests: count(MergeRequest),
notes: count(Note)
}.merge(
- runners_usage,
integrations_usage,
user_preferences_usage,
service_desk_counts
@@ -156,18 +147,6 @@ module Gitlab
end
# rubocop: enable Metrics/AbcSize
- def runners_usage
- {
- ci_runners: count(::Ci::Runner),
- ci_runners_instance_type_active: count(::Ci::Runner.instance_type.active),
- ci_runners_group_type_active: count(::Ci::Runner.group_type.active),
- ci_runners_project_type_active: count(::Ci::Runner.project_type.active),
- ci_runners_instance_type_active_online: count(::Ci::Runner.instance_type.active.online),
- ci_runners_group_type_active_online: count(::Ci::Runner.group_type.active.online),
- ci_runners_project_type_active_online: count(::Ci::Runner.project_type.active.online)
- }
- end
-
def system_usage_data_monthly
{
counts_monthly: {
@@ -240,7 +219,7 @@ module Gitlab
omniauth_enabled: alt_usage_data(fallback: nil) { Gitlab::Auth.omniauth_enabled? },
prometheus_enabled: alt_usage_data(fallback: nil) { Gitlab::Prometheus::Internal.prometheus_enabled? },
prometheus_metrics_enabled: alt_usage_data(fallback: nil) { Gitlab::Metrics.prometheus_metrics_enabled? },
- reply_by_email_enabled: alt_usage_data(fallback: nil) { Gitlab::IncomingEmail.enabled? },
+ reply_by_email_enabled: alt_usage_data(fallback: nil) { Gitlab::Email::IncomingEmail.enabled? },
web_ide_clientside_preview_enabled: alt_usage_data(fallback: nil) { false },
signup_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.allow_signup? },
grafana_link_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.grafana_enabled? },
@@ -333,24 +312,10 @@ module Gitlab
end
def jira_usage
- # Jira Cloud does not support custom domains as per https://jira.atlassian.com/browse/CLOUD-6999
- # so we can just check for subdomains of atlassian.net
- jira_integration_data_hash = jira_integration_data
- if jira_integration_data_hash.nil?
- return { projects_jira_server_active: FALLBACK, projects_jira_cloud_active: FALLBACK }
- end
-
- results = {
- projects_jira_server_active: 0,
- projects_jira_cloud_active: 0,
+ {
projects_jira_dvcs_cloud_active: count(ProjectFeatureUsage.with_jira_dvcs_integration_enabled),
projects_jira_dvcs_server_active: count(ProjectFeatureUsage.with_jira_dvcs_integration_enabled(cloud: false))
}
-
- results[:projects_jira_server_active] = jira_integration_data_hash[:projects_jira_server_active]
- results[:projects_jira_cloud_active] = jira_integration_data_hash[:projects_jira_cloud_active]
-
- results
end
# rubocop: enable CodeReuse/ActiveRecord
@@ -384,33 +349,13 @@ module Gitlab
}
end
- def merge_requests_users(time_period)
- counter = Gitlab::UsageDataCounters::TrackUniqueEvents
-
- redis_usage_data do
- counter.count_unique_events(
- event_action: Gitlab::UsageDataCounters::TrackUniqueEvents::MERGE_REQUEST_ACTION,
- date_from: time_period[:created_at].first,
- date_to: time_period[:created_at].last
- )
- end
- end
-
- def installation_type
- if Rails.env.production?
- Gitlab::INSTALLATION_TYPE
- else
- "gitlab-development-kit"
- end
- end
-
def operating_system
ohai_data = Ohai::System.new.tap do |oh|
oh.all_plugins(['platform'])
end.data
platform = ohai_data['platform']
- platform = 'raspbian' if ohai_data['platform'] == 'debian' && /armv/.match?(ohai_data['kernel']['machine'])
+ platform = 'raspbian' if ohai_data['platform'] == 'debian' && ohai_data['kernel']['machine']&.include?('armv')
"#{platform}-#{ohai_data['platform_version']}"
end
@@ -463,12 +408,7 @@ module Gitlab
projects_without_disable_overriding_approvers_per_merge_request: count(::Project.where(time_period.merge(disable_overriding_approvers_per_merge_request: [false, nil]))),
remote_mirrors: distinct_count(::Project.with_remote_mirrors.where(time_period), :creator_id),
snippets: distinct_count(::Snippet.where(time_period), :author_id)
- }.tap do |h|
- if time_period.present?
- h[:merge_requests_users] = merge_requests_users(time_period)
- h.merge!(action_monthly_active_users(time_period))
- end
- end
+ }
end
# rubocop: enable CodeReuse/ActiveRecord
@@ -527,7 +467,6 @@ module Gitlab
# Omitted because no user, creator or author associated: `boards`, `labels`, `milestones`, `uploads`
# Omitted because too expensive: `epics_deepest_relationship_level`
- # Omitted because of encrypted properties: `projects_jira_cloud_active`, `projects_jira_server_active`
# rubocop: disable CodeReuse/ActiveRecord
def usage_activity_by_stage_plan(time_period)
time_frame = metric_time_period(time_period)
@@ -582,17 +521,6 @@ module Gitlab
{}
end
- def action_monthly_active_users(time_period)
- counter = Gitlab::UsageDataCounters::EditorUniqueCounter
- date_range = { date_from: time_period[:created_at].first, date_to: time_period[:created_at].last }
-
- {
- action_monthly_active_users_web_ide_edit: redis_usage_data { counter.count_web_ide_edit_actions(**date_range) },
- action_monthly_active_users_sfe_edit: redis_usage_data { counter.count_sfe_edit_actions(**date_range) },
- action_monthly_active_users_snippet_editor_edit: redis_usage_data { counter.count_snippet_editor_edit_actions(**date_range) }
- }
- end
-
def with_metadata
result = nil
error = nil
@@ -717,7 +645,6 @@ module Gitlab
time_frame = metric_time_period(time_period)
counters = {
gitlab_project: add_metric('CountImportedProjectsMetric', time_frame: time_frame, options: { import_type: 'gitlab_project' }),
- gitlab: add_metric('CountImportedProjectsMetric', time_frame: time_frame, options: { import_type: 'gitlab' }),
github: add_metric('CountImportedProjectsMetric', time_frame: time_frame, options: { import_type: 'github' }),
bitbucket: add_metric('CountImportedProjectsMetric', time_frame: time_frame, options: { import_type: 'bitbucket' }),
bitbucket_server: add_metric('CountImportedProjectsMetric', time_frame: time_frame, options: { import_type: 'bitbucket_server' }),
@@ -737,7 +664,6 @@ module Gitlab
{
jira: count(::JiraImportState.where(time_period)), # rubocop: disable CodeReuse/ActiveRecord
fogbugz: add_metric('CountImportedProjectsMetric', time_frame: time_frame, options: { import_type: 'fogbugz' }),
- phabricator: add_metric('CountImportedProjectsMetric', time_frame: time_frame, options: { import_type: 'phabricator' }),
csv: count(::Issues::CsvImport.where(time_period)) # rubocop: disable CodeReuse/ActiveRecord
}
end
diff --git a/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb b/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb
index 7f6d67e01c7..97091ff975b 100644
--- a/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb
+++ b/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb
@@ -2,7 +2,7 @@
module Gitlab::UsageDataCounters
class CiTemplateUniqueCounter
- REDIS_SLOT = 'ci_templates'
+ PREFIX = 'ci_templates'
KNOWN_EVENTS_FILE_PATH = File.expand_path('known_events/ci_templates.yml', __dir__)
class << self
@@ -28,7 +28,7 @@ module Gitlab::UsageDataCounters
def ci_template_event_name(template_name, config_source)
prefix = 'implicit_' if config_source.to_s == 'auto_devops_source'
- "p_#{REDIS_SLOT}_#{prefix}#{template_to_event_name(template_name)}"
+ "p_#{PREFIX}_#{prefix}#{template_to_event_name(template_name)}"
end
def expand_template_name(template_name)
diff --git a/lib/gitlab/usage_data_counters/container_registry_event_counter.rb b/lib/gitlab/usage_data_counters/container_registry_event_counter.rb
new file mode 100644
index 00000000000..5d54bb18443
--- /dev/null
+++ b/lib/gitlab/usage_data_counters/container_registry_event_counter.rb
@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module UsageDataCounters
+ class ContainerRegistryEventCounter < BaseCounter
+ KNOWN_EVENTS = %w[i_container_registry_delete_manifest].freeze
+ PREFIX = 'container_registry_events'
+ end
+ end
+end
diff --git a/lib/gitlab/usage_data_counters/counter_events/package_events.yml b/lib/gitlab/usage_data_counters/counter_events/package_events.yml
index f7ddc53f50d..129bf77c7f0 100644
--- a/lib/gitlab/usage_data_counters/counter_events/package_events.yml
+++ b/lib/gitlab/usage_data_counters/counter_events/package_events.yml
@@ -7,6 +7,7 @@
- i_package_conan_push_package
- i_package_debian_delete_package
- i_package_debian_pull_package
+- i_package_debian_push_package
- i_package_delete_package
- i_package_delete_package_by_deploy_token
- i_package_delete_package_by_guest
diff --git a/lib/gitlab/usage_data_counters/editor_unique_counter.rb b/lib/gitlab/usage_data_counters/editor_unique_counter.rb
index 2aebc1b8813..4e4a01ed301 100644
--- a/lib/gitlab/usage_data_counters/editor_unique_counter.rb
+++ b/lib/gitlab/usage_data_counters/editor_unique_counter.rb
@@ -38,18 +38,16 @@ module Gitlab
def track_unique_action(event_name, author, time, project = nil)
return unless author
- if Feature.enabled?(:route_hll_to_snowplow_phase2)
- Gitlab::Tracking.event(
- name,
- 'ide_edit',
- property: event_name.to_s,
- project: project,
- namespace: project&.namespace,
- user: author,
- label: 'usage_activity_by_stage_monthly.create.action_monthly_active_users_ide_edit',
- context: [Gitlab::Tracking::ServicePingContext.new(data_source: :redis_hll, event: event_name).to_context]
- )
- end
+ Gitlab::Tracking.event(
+ name,
+ 'ide_edit',
+ property: event_name.to_s,
+ project: project,
+ namespace: project&.namespace,
+ user: author,
+ label: 'usage_activity_by_stage_monthly.create.action_monthly_active_users_ide_edit',
+ context: [Gitlab::Tracking::ServicePingContext.new(data_source: :redis_hll, event: event_name).to_context]
+ )
Gitlab::UsageDataCounters::HLLRedisCounter.track_event(event_name, values: author.id, time: time)
end
diff --git a/lib/gitlab/usage_data_counters/gitlab_cli_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/gitlab_cli_activity_unique_counter.rb
index 8a57a0331b8..b30c4b675f9 100644
--- a/lib/gitlab/usage_data_counters/gitlab_cli_activity_unique_counter.rb
+++ b/lib/gitlab/usage_data_counters/gitlab_cli_activity_unique_counter.rb
@@ -4,7 +4,9 @@ module Gitlab
module UsageDataCounters
module GitLabCliActivityUniqueCounter
GITLAB_CLI_API_REQUEST_ACTION = 'i_code_review_user_gitlab_cli_api_request'
- GITLAB_CLI_USER_AGENT_REGEX = /GitLab\sCLI$/.freeze
+
+ # This regex will match to user agents ending with GitLab CLI or starting with glab/v"
+ GITLAB_CLI_USER_AGENT_REGEX = %r{(GitLab\sCLI$|^glab/v)}.freeze
class << self
def track_api_request_when_trackable(user_agent:, user:)
diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb
index b809e6c4e42..7ddd496cd85 100644
--- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb
+++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb
@@ -5,26 +5,17 @@ module Gitlab
module HLLRedisCounter
DEFAULT_WEEKLY_KEY_EXPIRY_LENGTH = 6.weeks
DEFAULT_DAILY_KEY_EXPIRY_LENGTH = 29.days
- DEFAULT_REDIS_SLOT = ''
+ REDIS_SLOT = 'hll_counters'
EventError = Class.new(StandardError)
UnknownEvent = Class.new(EventError)
UnknownAggregation = Class.new(EventError)
AggregationMismatch = Class.new(EventError)
- SlotMismatch = Class.new(EventError)
- CategoryMismatch = Class.new(EventError)
InvalidContext = Class.new(EventError)
KNOWN_EVENTS_PATH = File.expand_path('known_events/*.yml', __dir__)
ALLOWED_AGGREGATIONS = %i(daily weekly).freeze
- CATEGORIES_FOR_TOTALS = %w[
- compliance
- error_tracking
- ide_edit
- pipeline_authoring
- ].freeze
-
# Track event on entity_id
# Increment a Redis HLL counter for unique event_name and entity_id
#
@@ -33,10 +24,7 @@ module Gitlab
# Event example:
#
# - name: g_compliance_dashboard # Unique event name
- # redis_slot: compliance # Optional slot name, if not defined it will use name as a slot, used for totals
- # category: compliance # Group events in categories
- # aggregation: daily # Aggregation level, keys are stored daily or weekly
- # feature_flag: # The event feature flag
+ # aggregation: weekly # Aggregation level, keys are stored weekly
#
# Usage:
#
@@ -76,23 +64,11 @@ module Gitlab
# context - Event context, plan level tracking. Available if set when tracking.
def unique_events(event_names:, start_date:, end_date:, context: '')
count_unique_events(event_names: event_names, start_date: start_date, end_date: end_date, context: context) do |events|
- raise SlotMismatch, events unless events_in_same_slot?(events)
- raise CategoryMismatch, events unless events_in_same_category?(events)
raise AggregationMismatch, events unless events_same_aggregation?(events)
raise InvalidContext if context.present? && !context.in?(valid_context_list)
end
end
- def categories
- @categories ||= known_events.map { |event| event[:category] }.uniq
- end
-
- # @param category [String] the category name
- # @return [Array<String>] list of event names for given category
- def events_for_category(category)
- known_events.select { |event| event[:category] == category.to_s }.map { |event| event[:name] }
- end
-
def known_event?(event_name)
event_for(event_name).present?
end
@@ -103,7 +79,6 @@ module Gitlab
def calculate_events_union(event_names:, start_date:, end_date:)
count_unique_events(event_names: event_names, start_date: start_date, end_date: end_date) do |events|
- raise SlotMismatch, events unless events_in_same_slot?(events)
raise AggregationMismatch, events unless events_same_aggregation?(events)
end
end
@@ -117,9 +92,15 @@ module Gitlab
Gitlab::ErrorTracking.track_and_raise_for_dev_exception(UnknownEvent.new("Unknown event #{event_name}")) unless event.present?
return if event.blank?
- return unless feature_enabled?(event)
+ return unless Feature.enabled?(:redis_hll_tracking, type: :ops)
+
+ if event[:aggregation].to_sym == :daily
+ weekly_event = event.dup.tap { |e| e['aggregation'] = 'weekly' }
+ Gitlab::Redis::HLL.add(key: redis_key(weekly_event, time, context), value: values, expiry: expiry(weekly_event))
+ end
Gitlab::Redis::HLL.add(key: redis_key(event, time, context), value: values, expiry: expiry(event))
+
rescue StandardError => e
# Ignore any exceptions unless is dev or test env
# The application flow should not be blocked by erros in tracking
@@ -138,6 +119,11 @@ module Gitlab
aggregation = events.first[:aggregation]
+ if Feature.disabled?(:revert_daily_hll_events_to_weekly_aggregation)
+ aggregation = :weekly
+ events.each { |e| e[:aggregation] = :weekly }
+ end
+
keys = keys_for_aggregation(aggregation, events: events, start_date: start_date, end_date: end_date, context: context)
return FALLBACK unless keys.any?
@@ -145,21 +131,6 @@ module Gitlab
redis_usage_data { Gitlab::Redis::HLL.count(keys: keys) }
end
- def feature_enabled?(event)
- return true if event[:feature_flag].blank?
-
- Feature.enabled?(event[:feature_flag]) && Feature.enabled?(:redis_hll_tracking, type: :ops)
- end
-
- # Allow to add totals for events that are in the same redis slot, category and have the same aggregation level
- # and if there are more than 1 event
- def eligible_for_totals?(events_names)
- return false if events_names.size <= 1
-
- events = events_for(events_names)
- events_in_same_slot?(events) && events_in_same_category?(events) && events_same_aggregation?(events)
- end
-
def keys_for_aggregation(aggregation, events:, start_date:, end_date:, context: '')
if aggregation.to_sym == :daily
daily_redis_keys(events: events, start_date: start_date, end_date: end_date, context: context)
@@ -182,20 +153,6 @@ module Gitlab
known_events.map { |event| event[:name] }
end
- def events_in_same_slot?(events)
- # if we check one event then redis_slot is only one to check
- return false if events.empty?
- return true if events.size == 1
-
- slot = events.first[:redis_slot]
- events.all? { |event| event[:redis_slot].present? && event[:redis_slot] == slot }
- end
-
- def events_in_same_category?(events)
- category = events.first[:category]
- events.all? { |event| event[:category] == category }
- end
-
def events_same_aggregation?(events)
aggregation = events.first[:aggregation]
events.all? { |event| event[:aggregation] == aggregation }
@@ -213,30 +170,19 @@ module Gitlab
known_events.select { |event| event_names.include?(event[:name]) }
end
- def redis_slot(event)
- event[:redis_slot] || DEFAULT_REDIS_SLOT
- end
-
# Compose the key in order to store events daily or weekly
def redis_key(event, time, context = '')
raise UnknownEvent, "Unknown event #{event[:name]}" unless known_events_names.include?(event[:name].to_s)
+
+ # ToDo: remove during https://gitlab.com/groups/gitlab-org/-/epics/9542 cleanup
raise UnknownAggregation, "Use :daily or :weekly aggregation" unless ALLOWED_AGGREGATIONS.include?(event[:aggregation].to_sym)
- key = apply_slot(event)
+ key = "{#{REDIS_SLOT}}_#{event[:name]}"
key = apply_time_aggregation(key, time, event)
key = "#{context}_#{key}" if context.present?
key
end
- def apply_slot(event)
- slot = redis_slot(event)
- if slot.present?
- event[:name].to_s.gsub(slot, "{#{slot}}")
- else
- "{#{event[:name]}}"
- end
- end
-
def apply_time_aggregation(key, time, event)
if event[:aggregation].to_sym == :daily
year_day = time.strftime('%G-%j')
diff --git a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb
index a59ea36961d..31f090e0f51 100644
--- a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb
+++ b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb
@@ -37,8 +37,8 @@ module Gitlab
ISSUE_DESIGN_COMMENT_REMOVED = 'g_project_management_issue_design_comments_removed'
class << self
- def track_issue_created_action(author:, project:)
- track_snowplow_action(ISSUE_CREATED, author, project)
+ def track_issue_created_action(author:, namespace:)
+ track_snowplow_action(ISSUE_CREATED, author, namespace)
track_unique_action(ISSUE_CREATED, author)
end
@@ -179,8 +179,16 @@ module Gitlab
private
- def track_snowplow_action(event_name, author, project)
- return unless Feature.enabled?(:route_hll_to_snowplow_phase2, project.namespace)
+ def track_snowplow_action(event_name, author, container)
+ namespace, project = case container
+ when Project
+ [container.namespace, container]
+ when Namespaces::ProjectNamespace
+ [container.parent, container.project]
+ else
+ [container, nil]
+ end
+
return unless author
Gitlab::Tracking.event(
@@ -189,7 +197,7 @@ module Gitlab
label: ISSUE_LABEL,
property: event_name,
project: project,
- namespace: project.namespace,
+ namespace: namespace,
user: author,
context: [Gitlab::Tracking::ServicePingContext.new(data_source: :redis_hll, event: event_name).to_context]
)
diff --git a/lib/gitlab/usage_data_counters/known_events/analytics.yml b/lib/gitlab/usage_data_counters/known_events/analytics.yml
index 85524c766ca..0b30308b552 100644
--- a/lib/gitlab/usage_data_counters/known_events/analytics.yml
+++ b/lib/gitlab/usage_data_counters/known_events/analytics.yml
@@ -1,52 +1,26 @@
- name: users_viewing_analytics_group_devops_adoption
- category: analytics
- redis_slot: analytics
aggregation: weekly
- name: i_analytics_dev_ops_adoption
- category: analytics
- redis_slot: analytics
aggregation: weekly
- name: i_analytics_dev_ops_score
- category: analytics
- redis_slot: analytics
aggregation: weekly
- name: i_analytics_instance_statistics
- category: analytics
- redis_slot: analytics
aggregation: weekly
- name: p_analytics_pipelines
- category: analytics
- redis_slot: analytics
aggregation: weekly
- name: p_analytics_valuestream
- category: analytics
- redis_slot: analytics
aggregation: weekly
- name: p_analytics_repo
- category: analytics
- redis_slot: analytics
aggregation: weekly
- name: i_analytics_cohorts
- category: analytics
- redis_slot: analytics
aggregation: weekly
- name: p_analytics_ci_cd_pipelines
- category: analytics
- redis_slot: analytics
aggregation: weekly
- name: p_analytics_ci_cd_deployment_frequency
- category: analytics
- redis_slot: analytics
aggregation: weekly
- name: p_analytics_ci_cd_lead_time
- category: analytics
- redis_slot: analytics
aggregation: weekly
- name: p_analytics_ci_cd_time_to_restore_service
- category: analytics
- redis_slot: analytics
aggregation: weekly
- name: p_analytics_ci_cd_change_failure_rate
- category: analytics
- redis_slot: analytics
aggregation: weekly
diff --git a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml
index b13e3d631c7..f685f0d65d9 100644
--- a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml
+++ b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml
@@ -4,602 +4,306 @@
# Do not edit it manually!
---
- name: p_ci_templates_terraform_base_latest
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_terraform_base
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_dotnet
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_nodejs
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_openshift
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_auto_devops
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_bash
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_rust
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_elixir
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_clojure
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_crystal
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_getting_started
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_code_quality
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_verify_load_performance_testing
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_verify_accessibility
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_verify_failfast
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_verify_browser_performance
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_verify_browser_performance_latest
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_grails
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_security_sast
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_security_dast_runner_validation
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_security_dast_on_demand_scan
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_security_secret_detection
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_security_license_scanning
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_security_coverage_fuzzing_latest
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_security_dast_on_demand_api_scan
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_security_coverage_fuzzing
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_security_api_fuzzing_latest
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_security_secure_binaries
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_security_dast_api
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_security_container_scanning
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_security_dast_latest
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_security_sast_iac
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_security_dependency_scanning
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_security_dast_api_latest
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_security_container_scanning_latest
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_security_api_fuzzing
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_security_dast
- category: ci_templates
- redis_slot: ci_templates
+ aggregation: weekly
+- name: p_ci_templates_security_api_discovery
aggregation: weekly
- name: p_ci_templates_security_fortify_fod_sast
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_security_sast_iac_latest
- category: ci_templates
- redis_slot: ci_templates
+ aggregation: weekly
+- name: p_ci_templates_security_bas_latest
aggregation: weekly
- name: p_ci_templates_qualys_iac_security
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_ios_fastlane
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_composer
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_c
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_python
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_android_fastlane
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_android_latest
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_django
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_maven
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_liquibase
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_flutter
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_workflows_branch_pipelines
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_workflows_mergerequest_pipelines
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_laravel
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_kaniko
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_php
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_packer
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_themekit
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_terraform
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_katalon
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_mono
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_go
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_scala
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_latex
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_android
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_indeni_cloudrail
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_matlab
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_deploy_ecs
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_aws_cf_provision_and_deploy_ec2
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_aws_deploy_ecs
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_gradle
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_chef
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_jobs_dast_default_branch_deploy
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_jobs_load_performance_testing
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_jobs_helm_2to3
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_jobs_sast
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_jobs_secret_detection
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_jobs_license_scanning
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_jobs_code_intelligence
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_jobs_code_quality
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_jobs_deploy_ecs
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_jobs_deploy_ec2
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_jobs_license_scanning_latest
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_jobs_deploy
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_jobs_build
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_jobs_browser_performance_testing
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_jobs_container_scanning
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_jobs_container_scanning_latest
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_jobs_dependency_scanning_latest
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_jobs_test
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_jobs_sast_latest
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_jobs_sast_iac
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_jobs_secret_detection_latest
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_jobs_dependency_scanning
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_jobs_deploy_latest
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_jobs_browser_performance_testing_latest
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_jobs_cf_provision
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_jobs_build_latest
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_jobs_sast_iac_latest
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_terraform_latest
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_swift
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_pages_jekyll
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_pages_harp
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_pages_octopress
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_pages_brunch
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_pages_doxygen
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_pages_hyde
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_pages_lektor
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_pages_jbake
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_pages_hexo
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_pages_middleman
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_pages_hugo
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_pages_pelican
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_pages_nanoc
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_pages_swaggerui
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_pages_jigsaw
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_pages_metalsmith
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_pages_gatsby
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_pages_html
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_dart
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_docker
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_julia
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_npm
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_dotnet_core
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_5_minute_production_app
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_ruby
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_implicit_auto_devops
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_implicit_jobs_browser_performance_testing
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_implicit_jobs_build
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_implicit_jobs_code_intelligence
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_implicit_jobs_code_quality
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_implicit_jobs_container_scanning
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_implicit_jobs_dast_default_branch_deploy
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_implicit_jobs_dependency_scanning
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_implicit_jobs_deploy
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_implicit_jobs_deploy_ec2
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_implicit_jobs_deploy_ecs
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_implicit_jobs_helm_2to3
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_implicit_jobs_license_scanning
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_implicit_jobs_sast
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_implicit_jobs_secret_detection
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_implicit_jobs_test
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_implicit_security_container_scanning
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_implicit_security_dast
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_implicit_security_dependency_scanning
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_implicit_security_license_scanning
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_implicit_security_sast
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_implicit_security_secret_detection
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_terraform_module_base
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_terraform_module
- category: ci_templates
- redis_slot: ci_templates
aggregation: weekly
diff --git a/lib/gitlab/usage_data_counters/known_events/ci_users.yml b/lib/gitlab/usage_data_counters/known_events/ci_users.yml
index b012d61eef5..49757c6e672 100644
--- a/lib/gitlab/usage_data_counters/known_events/ci_users.yml
+++ b/lib/gitlab/usage_data_counters/known_events/ci_users.yml
@@ -1,10 +1,4 @@
- name: ci_users_executing_deployment_job
- category: ci_users
- redis_slot: ci_users
aggregation: weekly
- feature_flag:
- name: ci_users_executing_verify_environment_job
- category: ci_users
- redis_slot: ci_users
aggregation: weekly
- feature_flag:
diff --git a/lib/gitlab/usage_data_counters/known_events/code_review_events.yml b/lib/gitlab/usage_data_counters/known_events/code_review_events.yml
index 3bb6655d762..db0c0653f63 100644
--- a/lib/gitlab/usage_data_counters/known_events/code_review_events.yml
+++ b/lib/gitlab/usage_data_counters/known_events/code_review_events.yml
@@ -1,457 +1,233 @@
---
- name: i_code_review_create_note_in_ipynb_diff
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_create_note_in_ipynb_diff_mr
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_create_note_in_ipynb_diff_commit
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_user_create_note_in_ipynb_diff
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_user_create_note_in_ipynb_diff_mr
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_user_create_note_in_ipynb_diff_commit
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_mr_diffs
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_user_single_file_diffs
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_mr_single_file_diffs
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_user_toggled_task_item_status
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_create_mr
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_user_create_mr
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_user_close_mr
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_user_reopen_mr
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_user_approve_mr
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_user_unapprove_mr
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_user_resolve_thread
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_user_unresolve_thread
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_edit_mr_title
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_edit_mr_desc
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_user_merge_mr
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_user_create_mr_comment
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_user_edit_mr_comment
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_user_remove_mr_comment
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_user_create_review_note
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_user_publish_review
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_user_create_multiline_mr_comment
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_user_edit_multiline_mr_comment
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_user_remove_multiline_mr_comment
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_user_add_suggestion
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_user_apply_suggestion
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_user_assigned
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_user_marked_as_draft
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_user_unmarked_as_draft
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_user_review_requested
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_user_approval_rule_added
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_user_approval_rule_deleted
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_user_approval_rule_edited
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_user_vs_code_api_request
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_user_jetbrains_api_request
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_user_gitlab_cli_api_request
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_user_create_mr_from_issue
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_user_mr_discussion_locked
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_user_mr_discussion_unlocked
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_user_time_estimate_changed
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_user_time_spent_changed
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_user_assignees_changed
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_user_reviewers_changed
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_user_milestone_changed
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_user_labels_changed
- redis_slot: code_review
- category: code_review
aggregation: weekly
# Diff settings events
- name: i_code_review_click_diff_view_setting
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_click_single_file_mode_setting
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_click_file_browser_setting
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_click_whitespace_setting
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_diff_view_inline
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_diff_view_parallel
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_file_browser_tree_view
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_file_browser_list_view
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_diff_show_whitespace
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_diff_hide_whitespace
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_diff_single_file
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_diff_multiple_files
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_user_load_conflict_ui
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_user_resolve_conflict
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_user_searches_diff
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_total_suggestions_applied
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_total_suggestions_added
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_user_resolve_thread_in_issue
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_widget_nothing_merge_click_new_file
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_post_merge_delete_branch
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_post_merge_click_revert
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_post_merge_click_cherry_pick
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_post_merge_submit_revert_modal
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_post_merge_submit_cherry_pick_modal
- redis_slot: code_review
- category: code_review
aggregation: weekly
# MR Widget Extensions
## Test Summary
- name: i_code_review_merge_request_widget_test_summary_view
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_merge_request_widget_test_summary_full_report_clicked
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_merge_request_widget_test_summary_expand
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_merge_request_widget_test_summary_expand_success
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_merge_request_widget_test_summary_expand_warning
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_merge_request_widget_test_summary_expand_failed
- redis_slot: code_review
- category: code_review
aggregation: weekly
## Accessibility
- name: i_code_review_merge_request_widget_accessibility_view
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_merge_request_widget_accessibility_full_report_clicked
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_merge_request_widget_accessibility_expand
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_merge_request_widget_accessibility_expand_success
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_merge_request_widget_accessibility_expand_warning
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_merge_request_widget_accessibility_expand_failed
- redis_slot: code_review
- category: code_review
aggregation: weekly
## Code Quality
- name: i_code_review_merge_request_widget_code_quality_view
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_merge_request_widget_code_quality_full_report_clicked
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_merge_request_widget_code_quality_expand
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_merge_request_widget_code_quality_expand_success
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_merge_request_widget_code_quality_expand_warning
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_merge_request_widget_code_quality_expand_failed
- redis_slot: code_review
- category: code_review
aggregation: weekly
## Terraform
- name: i_code_review_merge_request_widget_terraform_view
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_merge_request_widget_terraform_full_report_clicked
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_merge_request_widget_terraform_expand
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_merge_request_widget_terraform_expand_success
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_merge_request_widget_terraform_expand_warning
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_merge_request_widget_terraform_expand_failed
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_submit_review_approve
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_submit_review_comment
- redis_slot: code_review
- category: code_review
aggregation: weekly
## License Compliance
- name: i_code_review_merge_request_widget_license_compliance_view
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_merge_request_widget_license_compliance_full_report_clicked
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_merge_request_widget_license_compliance_expand
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_merge_request_widget_license_compliance_expand_success
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_merge_request_widget_license_compliance_expand_warning
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_merge_request_widget_license_compliance_expand_failed
- redis_slot: code_review
- category: code_review
aggregation: weekly
## Security Reports
- name: i_code_review_merge_request_widget_security_reports_view
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_merge_request_widget_security_reports_full_report_clicked
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_merge_request_widget_security_reports_expand
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_merge_request_widget_security_reports_expand_success
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_merge_request_widget_security_reports_expand_warning
- redis_slot: code_review
- category: code_review
aggregation: weekly
- name: i_code_review_merge_request_widget_security_reports_expand_failed
- redis_slot: code_review
- category: code_review
aggregation: weekly
diff --git a/lib/gitlab/usage_data_counters/known_events/common.yml b/lib/gitlab/usage_data_counters/known_events/common.yml
index ae15530f0d0..0583d85c3cc 100644
--- a/lib/gitlab/usage_data_counters/known_events/common.yml
+++ b/lib/gitlab/usage_data_counters/known_events/common.yml
@@ -1,313 +1,167 @@
---
# Compliance category
- name: g_edit_by_web_ide
- category: ide_edit
- redis_slot: edit
aggregation: daily
- name: g_edit_by_sfe
- category: ide_edit
- redis_slot: edit
aggregation: daily
- name: g_edit_by_snippet_ide
- category: ide_edit
- redis_slot: edit
aggregation: daily
- name: g_edit_by_live_preview
- category: ide_edit
- redis_slot: edit
aggregation: daily
- name: i_search_total
- category: search
- redis_slot: search
aggregation: weekly
- name: wiki_action
- category: source_code
aggregation: daily
- name: design_action
- category: source_code
aggregation: daily
- name: project_action
- category: source_code
aggregation: daily
- name: git_write_action
- category: source_code
aggregation: daily
- name: merge_request_action
- category: source_code
aggregation: daily
- name: i_source_code_code_intelligence
- redis_slot: source_code
- category: source_code
aggregation: daily
# Incident management
- name: incident_management_alert_status_changed
- redis_slot: incident_management
- category: incident_management
aggregation: weekly
- name: incident_management_alert_assigned
- redis_slot: incident_management
- category: incident_management
aggregation: weekly
- name: incident_management_alert_todo
- redis_slot: incident_management
- category: incident_management
aggregation: weekly
- name: incident_management_incident_created
- redis_slot: incident_management
- category: incident_management
aggregation: weekly
- name: incident_management_incident_reopened
- redis_slot: incident_management
- category: incident_management
aggregation: weekly
- name: incident_management_incident_closed
- redis_slot: incident_management
- category: incident_management
aggregation: weekly
- name: incident_management_incident_assigned
- redis_slot: incident_management
- category: incident_management
aggregation: weekly
- name: incident_management_incident_todo
- redis_slot: incident_management
- category: incident_management
aggregation: weekly
- name: incident_management_incident_comment
- redis_slot: incident_management
- category: incident_management
aggregation: weekly
- name: incident_management_incident_zoom_meeting
- redis_slot: incident_management
- category: incident_management
aggregation: weekly
- name: incident_management_incident_relate
- redis_slot: incident_management
- category: incident_management
aggregation: weekly
- name: incident_management_incident_unrelate
- redis_slot: incident_management
- category: incident_management
aggregation: weekly
- name: incident_management_incident_change_confidential
- redis_slot: incident_management
- category: incident_management
aggregation: weekly
# Incident management timeline events
- name: incident_management_timeline_event_created
- redis_slot: incident_management
- category: incident_management
aggregation: weekly
- name: incident_management_timeline_event_edited
- redis_slot: incident_management
- category: incident_management
aggregation: weekly
- name: incident_management_timeline_event_deleted
- redis_slot: incident_management
- category: incident_management
aggregation: weekly
# Incident management alerts
- name: incident_management_alert_create_incident
- redis_slot: incident_management
- category: incident_management_alerts
aggregation: weekly
# Testing category
- name: i_testing_test_case_parsed
- category: testing
- redis_slot: testing
- aggregation: weekly
-- name: i_testing_summary_widget_total
- category: testing
- redis_slot: testing
aggregation: weekly
- name: i_testing_test_report_uploaded
- category: testing
- redis_slot: testing
aggregation: weekly
- name: i_testing_coverage_report_uploaded
- category: testing
- redis_slot: testing
aggregation: weekly
# Project Management group
- name: g_project_management_issue_title_changed
- category: issues_edit
- redis_slot: project_management
aggregation: daily
- name: g_project_management_issue_description_changed
- category: issues_edit
- redis_slot: project_management
aggregation: daily
- name: g_project_management_issue_assignee_changed
- category: issues_edit
- redis_slot: project_management
aggregation: daily
- name: g_project_management_issue_made_confidential
- category: issues_edit
- redis_slot: project_management
aggregation: daily
- name: g_project_management_issue_made_visible
- category: issues_edit
- redis_slot: project_management
aggregation: daily
- name: g_project_management_issue_created
- category: issues_edit
- redis_slot: project_management
aggregation: daily
- name: g_project_management_issue_closed
- category: issues_edit
- redis_slot: project_management
aggregation: daily
- name: g_project_management_issue_reopened
- category: issues_edit
- redis_slot: project_management
aggregation: daily
- name: g_project_management_issue_label_changed
- category: issues_edit
- redis_slot: project_management
aggregation: daily
- name: g_project_management_issue_milestone_changed
- category: issues_edit
- redis_slot: project_management
aggregation: daily
- name: g_project_management_issue_cross_referenced
- category: issues_edit
- redis_slot: project_management
aggregation: daily
- name: g_project_management_issue_moved
- category: issues_edit
- redis_slot: project_management
aggregation: daily
- name: g_project_management_issue_related
- category: issues_edit
- redis_slot: project_management
aggregation: daily
- name: g_project_management_issue_unrelated
- category: issues_edit
- redis_slot: project_management
aggregation: daily
- name: g_project_management_issue_marked_as_duplicate
- category: issues_edit
- redis_slot: project_management
aggregation: daily
- name: g_project_management_issue_locked
- category: issues_edit
- redis_slot: project_management
aggregation: daily
- name: g_project_management_issue_unlocked
- category: issues_edit
- redis_slot: project_management
aggregation: daily
- name: g_project_management_issue_designs_added
- category: issues_edit
- redis_slot: project_management
aggregation: daily
- name: g_project_management_issue_designs_modified
- category: issues_edit
- redis_slot: project_management
aggregation: daily
- name: g_project_management_issue_designs_removed
- category: issues_edit
- redis_slot: project_management
aggregation: daily
- name: g_project_management_issue_due_date_changed
- category: issues_edit
- redis_slot: project_management
aggregation: daily
- name: g_project_management_issue_design_comments_removed
- category: issues_edit
- redis_slot: project_management
aggregation: daily
- name: g_project_management_issue_time_estimate_changed
- category: issues_edit
- redis_slot: project_management
aggregation: daily
- name: g_project_management_issue_time_spent_changed
- category: issues_edit
- redis_slot: project_management
aggregation: daily
- name: g_project_management_issue_comment_added
- category: issues_edit
- redis_slot: project_management
aggregation: daily
- name: g_project_management_issue_comment_edited
- category: issues_edit
- redis_slot: project_management
aggregation: daily
- name: g_project_management_issue_comment_removed
- category: issues_edit
- redis_slot: project_management
aggregation: daily
- name: g_project_management_issue_cloned
- category: issues_edit
- redis_slot: project_management
aggregation: daily
# Runner group
- name: g_runner_fleet_read_jobs_statistics
- category: runner
- redis_slot: runner
aggregation: weekly
# Secrets Management
- name: i_snippets_show
- category: snippets
- redis_slot: snippets
aggregation: weekly
# Terraform
- name: p_terraform_state_api_unique_users
- category: terraform
- redis_slot: terraform
aggregation: weekly
# Pipeline Authoring group
+- name: ci_interpolation_users
+ aggregation: weekly
- name: o_pipeline_authoring_unique_users_committing_ciconfigfile
- category: pipeline_authoring
- redis_slot: pipeline_authoring
aggregation: weekly
- name: o_pipeline_authoring_unique_users_pushing_mr_ciconfigfile
- category: pipeline_authoring
- redis_slot: pipeline_authoring
aggregation: weekly
- name: i_ci_secrets_management_id_tokens_build_created
- category: ci_secrets_management
- redis_slot: ci_secrets_management
aggregation: weekly
# Merge request widgets
- name: users_expanding_secure_security_report
- redis_slot: secure
- category: secure
aggregation: weekly
- name: users_expanding_testing_code_quality_report
- redis_slot: testing
- category: testing
aggregation: weekly
- name: users_expanding_testing_accessibility_report
- redis_slot: testing
- category: testing
aggregation: weekly
- name: users_expanding_testing_license_compliance_report
- redis_slot: testing
- category: testing
aggregation: weekly
- name: users_visiting_testing_license_compliance_full_report
- redis_slot: testing
- category: testing
aggregation: weekly
- name: users_visiting_testing_manage_license_compliance
- redis_slot: testing
- category: testing
aggregation: weekly
- name: users_clicking_license_testing_visiting_external_website
- redis_slot: testing
- category: testing
aggregation: weekly
# Geo group
- name: g_geo_proxied_requests
- category: geo
- redis_slot: geo
aggregation: daily
# Manage
- name: unique_active_user
- category: manage
aggregation: weekly
# Environments page
- name: users_visiting_environments_pages
- category: environments
- redis_slot: users
aggregation: weekly
diff --git a/lib/gitlab/usage_data_counters/known_events/container_registry_events.yml b/lib/gitlab/usage_data_counters/known_events/container_registry_events.yml
index e8b14de1769..aa0f9965fa7 100644
--- a/lib/gitlab/usage_data_counters/known_events/container_registry_events.yml
+++ b/lib/gitlab/usage_data_counters/known_events/container_registry_events.yml
@@ -1,22 +1,11 @@
---
- name: i_container_registry_push_tag_user
- category: user_container_registry
aggregation: weekly
- redis_slot: container_registry
- name: i_container_registry_delete_tag_user
- category: user_container_registry
aggregation: weekly
- redis_slot: container_registry
- name: i_container_registry_push_repository_user
- category: user_container_registry
aggregation: weekly
- redis_slot: container_registry
- name: i_container_registry_delete_repository_user
- category: user_container_registry
aggregation: weekly
- redis_slot: container_registry
- name: i_container_registry_create_repository_user
- category: user_container_registry
aggregation: weekly
- redis_slot: container_registry
- \ No newline at end of file
diff --git a/lib/gitlab/usage_data_counters/known_events/ecosystem.yml b/lib/gitlab/usage_data_counters/known_events/ecosystem.yml
index 7f7c9166086..6e4a893d19a 100644
--- a/lib/gitlab/usage_data_counters/known_events/ecosystem.yml
+++ b/lib/gitlab/usage_data_counters/known_events/ecosystem.yml
@@ -1,46 +1,24 @@
---
# Ecosystem category
- name: i_ecosystem_jira_service_close_issue
- category: ecosystem
- redis_slot: ecosystem
aggregation: weekly
- name: i_ecosystem_jira_service_cross_reference
- category: ecosystem
- redis_slot: ecosystem
aggregation: weekly
- name: i_ecosystem_slack_service_issue_notification
- category: ecosystem
- redis_slot: ecosystem
aggregation: weekly
- name: i_ecosystem_slack_service_push_notification
- category: ecosystem
- redis_slot: ecosystem
aggregation: weekly
- name: i_ecosystem_slack_service_deployment_notification
- category: ecosystem
- redis_slot: ecosystem
aggregation: weekly
- name: i_ecosystem_slack_service_wiki_page_notification
- category: ecosystem
- redis_slot: ecosystem
aggregation: weekly
- name: i_ecosystem_slack_service_merge_request_notification
- category: ecosystem
- redis_slot: ecosystem
aggregation: weekly
- name: i_ecosystem_slack_service_note_notification
- category: ecosystem
- redis_slot: ecosystem
aggregation: weekly
- name: i_ecosystem_slack_service_tag_push_notification
- category: ecosystem
- redis_slot: ecosystem
aggregation: weekly
- name: i_ecosystem_slack_service_confidential_note_notification
- category: ecosystem
- redis_slot: ecosystem
aggregation: weekly
- name: i_ecosystem_slack_service_confidential_issue_notification
- category: ecosystem
- redis_slot: ecosystem
aggregation: weekly
diff --git a/lib/gitlab/usage_data_counters/known_events/error_tracking.yml b/lib/gitlab/usage_data_counters/known_events/error_tracking.yml
index d80b711f8eb..ebfd1b274f9 100644
--- a/lib/gitlab/usage_data_counters/known_events/error_tracking.yml
+++ b/lib/gitlab/usage_data_counters/known_events/error_tracking.yml
@@ -1,9 +1,5 @@
---
- name: error_tracking_view_details
- category: error_tracking
- redis_slot: error_tracking
aggregation: weekly
- name: error_tracking_view_list
- category: error_tracking
- redis_slot: error_tracking
aggregation: weekly
diff --git a/lib/gitlab/usage_data_counters/known_events/importer_events.yml b/lib/gitlab/usage_data_counters/known_events/importer_events.yml
index c84d756a013..abbd83a012b 100644
--- a/lib/gitlab/usage_data_counters/known_events/importer_events.yml
+++ b/lib/gitlab/usage_data_counters/known_events/importer_events.yml
@@ -1,14 +1,13 @@
---
# Importer events
- name: github_import_project_start
- category: importer
- redis_slot: import
aggregation: weekly
- name: github_import_project_success
- category: importer
- redis_slot: import
aggregation: weekly
- name: github_import_project_failure
- category: importer
- redis_slot: import
aggregation: weekly
+- name: github_import_project_cancelled
+ aggregation: weekly
+- name: github_import_project_partially_completed
+ aggregation: weekly
+
diff --git a/lib/gitlab/usage_data_counters/known_events/integrations.yml b/lib/gitlab/usage_data_counters/known_events/integrations.yml
new file mode 100644
index 00000000000..4a83581e9f0
--- /dev/null
+++ b/lib/gitlab/usage_data_counters/known_events/integrations.yml
@@ -0,0 +1,18 @@
+- name: i_integrations_gitlab_for_slack_app_issue_notification
+ aggregation: weekly
+- name: i_integrations_gitlab_for_slack_app_push_notification
+ aggregation: weekly
+- name: i_integrations_gitlab_for_slack_app_deployment_notification
+ aggregation: weekly
+- name: i_integrations_gitlab_for_slack_app_wiki_page_notification
+ aggregation: weekly
+- name: i_integrations_gitlab_for_slack_app_merge_request_notification
+ aggregation: weekly
+- name: i_integrations_gitlab_for_slack_app_note_notification
+ aggregation: weekly
+- name: i_integrations_gitlab_for_slack_app_tag_push_notification
+ aggregation: weekly
+- name: i_integrations_gitlab_for_slack_app_confidential_note_notification
+ aggregation: weekly
+- name: i_integrations_gitlab_for_slack_app_confidential_issue_notification
+ aggregation: weekly
diff --git a/lib/gitlab/usage_data_counters/known_events/kubernetes_agent.yml b/lib/gitlab/usage_data_counters/known_events/kubernetes_agent.yml
index 966e6c584c7..b3d1c51c0e7 100644
--- a/lib/gitlab/usage_data_counters/known_events/kubernetes_agent.yml
+++ b/lib/gitlab/usage_data_counters/known_events/kubernetes_agent.yml
@@ -1,4 +1,2 @@
- name: agent_users_using_ci_tunnel
- category: kubernetes_agent
- redis_slot: agent
aggregation: weekly
diff --git a/lib/gitlab/usage_data_counters/known_events/package_events.yml b/lib/gitlab/usage_data_counters/known_events/package_events.yml
index ef8d02fa365..fa99798cde0 100644
--- a/lib/gitlab/usage_data_counters/known_events/package_events.yml
+++ b/lib/gitlab/usage_data_counters/known_events/package_events.yml
@@ -1,89 +1,49 @@
---
- name: i_package_composer_deploy_token
- category: deploy_token_packages
aggregation: weekly
- redis_slot: package
- name: i_package_composer_user
- category: user_packages
aggregation: weekly
- redis_slot: package
- name: i_package_conan_deploy_token
- category: deploy_token_packages
aggregation: weekly
- redis_slot: package
- name: i_package_conan_user
- category: user_packages
aggregation: weekly
- redis_slot: package
+- name: i_package_debian_deploy_token
+ aggregation: weekly
+- name: i_package_debian_user
+ aggregation: weekly
- name: i_package_generic_deploy_token
- category: deploy_token_packages
aggregation: weekly
- redis_slot: package
- name: i_package_generic_user
- category: user_packages
aggregation: weekly
- redis_slot: package
- name: i_package_helm_deploy_token
- category: deploy_token_packages
aggregation: weekly
- redis_slot: package
- name: i_package_helm_user
- category: user_packages
aggregation: weekly
- redis_slot: package
- name: i_package_maven_deploy_token
- category: deploy_token_packages
aggregation: weekly
- redis_slot: package
- name: i_package_maven_user
- category: user_packages
aggregation: weekly
- redis_slot: package
- name: i_package_npm_deploy_token
- category: deploy_token_packages
aggregation: weekly
- redis_slot: package
- name: i_package_npm_user
- category: user_packages
aggregation: weekly
- redis_slot: package
- name: i_package_nuget_deploy_token
- category: deploy_token_packages
aggregation: weekly
- redis_slot: package
- name: i_package_nuget_user
- category: user_packages
aggregation: weekly
- redis_slot: package
- name: i_package_pypi_deploy_token
- category: deploy_token_packages
aggregation: weekly
- redis_slot: package
- name: i_package_pypi_user
- category: user_packages
aggregation: weekly
- redis_slot: package
- name: i_package_rubygems_deploy_token
- category: deploy_token_packages
aggregation: weekly
- redis_slot: package
- name: i_package_rubygems_user
- category: user_packages
aggregation: weekly
- redis_slot: package
- name: i_package_terraform_module_deploy_token
- category: deploy_token_packages
aggregation: weekly
- redis_slot: package
- name: i_package_terraform_module_user
- category: user_packages
aggregation: weekly
- redis_slot: package
- name: i_package_rpm_user
- category: user_packages
aggregation: weekly
- redis_slot: package
- name: i_package_rpm_deploy_token
- category: deploy_token_packages
aggregation: weekly
- redis_slot: package
diff --git a/lib/gitlab/usage_data_counters/known_events/product_analytics.yml b/lib/gitlab/usage_data_counters/known_events/product_analytics.yml
new file mode 100644
index 00000000000..5a791c4b3c2
--- /dev/null
+++ b/lib/gitlab/usage_data_counters/known_events/product_analytics.yml
@@ -0,0 +1,4 @@
+- name: project_created_analytics_dashboard
+ aggregation: weekly
+- name: project_initialized_product_analytics
+ aggregation: weekly
diff --git a/lib/gitlab/usage_data_counters/known_events/quickactions.yml b/lib/gitlab/usage_data_counters/known_events/quickactions.yml
index 69b348b9a22..136d284f462 100644
--- a/lib/gitlab/usage_data_counters/known_events/quickactions.yml
+++ b/lib/gitlab/usage_data_counters/known_events/quickactions.yml
@@ -1,253 +1,135 @@
---
- name: i_quickactions_assign_multiple
- category: quickactions
- redis_slot: quickactions
aggregation: weekly
- name: i_quickactions_approve
- category: quickactions
- redis_slot: quickactions
aggregation: weekly
- name: i_quickactions_unapprove
- category: quickactions
- redis_slot: quickactions
aggregation: weekly
- name: i_quickactions_assign_single
- category: quickactions
- redis_slot: quickactions
aggregation: weekly
- name: i_quickactions_assign_self
- category: quickactions
- redis_slot: quickactions
aggregation: weekly
- name: i_quickactions_assign_reviewer
- category: quickactions
- redis_slot: quickactions
aggregation: weekly
- name: i_quickactions_award
- category: quickactions
- redis_slot: quickactions
aggregation: weekly
- name: i_quickactions_board_move
- category: quickactions
- redis_slot: quickactions
aggregation: weekly
- name: i_quickactions_clone
- category: quickactions
- redis_slot: quickactions
aggregation: weekly
- name: i_quickactions_close
- category: quickactions
- redis_slot: quickactions
aggregation: weekly
- name: i_quickactions_confidential
- category: quickactions
- redis_slot: quickactions
aggregation: weekly
- name: i_quickactions_copy_metadata_merge_request
- category: quickactions
- redis_slot: quickactions
aggregation: weekly
- name: i_quickactions_copy_metadata_issue
- category: quickactions
- redis_slot: quickactions
aggregation: weekly
- name: i_quickactions_create_merge_request
- category: quickactions
- redis_slot: quickactions
aggregation: weekly
- name: i_quickactions_done
- category: quickactions
- redis_slot: quickactions
aggregation: weekly
- name: i_quickactions_draft
- category: quickactions
- redis_slot: quickactions
aggregation: weekly
- name: i_quickactions_due
- category: quickactions
- redis_slot: quickactions
aggregation: weekly
- name: i_quickactions_duplicate
- category: quickactions
- redis_slot: quickactions
aggregation: weekly
- name: i_quickactions_estimate
- category: quickactions
- redis_slot: quickactions
aggregation: weekly
- name: i_quickactions_label
- category: quickactions
- redis_slot: quickactions
aggregation: weekly
- name: i_quickactions_lock
- category: quickactions
- redis_slot: quickactions
aggregation: weekly
- name: i_quickactions_merge
- category: quickactions
- redis_slot: quickactions
aggregation: weekly
- name: i_quickactions_milestone
- category: quickactions
- redis_slot: quickactions
aggregation: weekly
- name: i_quickactions_move
- category: quickactions
- redis_slot: quickactions
aggregation: weekly
- name: i_quickactions_promote_to_incident
- category: quickactions
- redis_slot: quickactions
aggregation: weekly
- name: i_quickactions_timeline
- category: quickactions
- redis_slot: quickactions
aggregation: weekly
- name: i_quickactions_ready
- category: quickactions
- redis_slot: quickactions
aggregation: weekly
- name: i_quickactions_reassign
- category: quickactions
- redis_slot: quickactions
aggregation: weekly
- name: i_quickactions_reassign_reviewer
- category: quickactions
- redis_slot: quickactions
aggregation: weekly
- name: i_quickactions_rebase
- category: quickactions
- redis_slot: quickactions
aggregation: weekly
- name: i_quickactions_relabel
- category: quickactions
- redis_slot: quickactions
aggregation: weekly
- name: i_quickactions_relate
- category: quickactions
- redis_slot: quickactions
aggregation: weekly
- name: i_quickactions_remove_due_date
- category: quickactions
- redis_slot: quickactions
aggregation: weekly
- name: i_quickactions_remove_estimate
- category: quickactions
- redis_slot: quickactions
aggregation: weekly
- name: i_quickactions_remove_milestone
- category: quickactions
- redis_slot: quickactions
aggregation: weekly
- name: i_quickactions_remove_time_spent
- category: quickactions
- redis_slot: quickactions
aggregation: weekly
- name: i_quickactions_remove_zoom
- category: quickactions
- redis_slot: quickactions
aggregation: weekly
- name: i_quickactions_reopen
- category: quickactions
- redis_slot: quickactions
aggregation: weekly
- name: i_quickactions_severity
- category: quickactions
- redis_slot: quickactions
aggregation: weekly
- name: i_quickactions_shrug
- category: quickactions
- redis_slot: quickactions
aggregation: weekly
- name: i_quickactions_spend_subtract
- category: quickactions
- redis_slot: quickactions
aggregation: weekly
- name: i_quickactions_spend_add
- category: quickactions
- redis_slot: quickactions
aggregation: weekly
- name: i_quickactions_submit_review
- category: quickactions
- redis_slot: quickactions
aggregation: weekly
- name: i_quickactions_subscribe
- category: quickactions
- redis_slot: quickactions
+ aggregation: weekly
+- name: i_quickactions_summarize_diff
aggregation: weekly
- name: i_quickactions_tableflip
- category: quickactions
- redis_slot: quickactions
aggregation: weekly
- name: i_quickactions_tag
- category: quickactions
- redis_slot: quickactions
aggregation: weekly
- name: i_quickactions_target_branch
- category: quickactions
- redis_slot: quickactions
aggregation: weekly
- name: i_quickactions_title
- category: quickactions
- redis_slot: quickactions
aggregation: weekly
- name: i_quickactions_todo
- category: quickactions
- redis_slot: quickactions
aggregation: weekly
- name: i_quickactions_unassign_specific
- category: quickactions
- redis_slot: quickactions
aggregation: weekly
- name: i_quickactions_unassign_all
- category: quickactions
- redis_slot: quickactions
aggregation: weekly
- name: i_quickactions_unassign_reviewer
- category: quickactions
- redis_slot: quickactions
aggregation: weekly
- name: i_quickactions_unlabel_specific
- category: quickactions
- redis_slot: quickactions
aggregation: weekly
- name: i_quickactions_unlabel_all
- category: quickactions
- redis_slot: quickactions
aggregation: weekly
- name: i_quickactions_unlock
- category: quickactions
- redis_slot: quickactions
aggregation: weekly
- name: i_quickactions_unsubscribe
- category: quickactions
- redis_slot: quickactions
aggregation: weekly
- name: i_quickactions_wip
- category: quickactions
- redis_slot: quickactions
aggregation: weekly
- name: i_quickactions_zoom
- category: quickactions
- redis_slot: quickactions
aggregation: weekly
- name: i_quickactions_link
- category: quickactions
- redis_slot: quickactions
aggregation: weekly
- name: i_quickactions_invite_email_single
- category: quickactions
- redis_slot: quickactions
aggregation: weekly
- name: i_quickactions_invite_email_multiple
- category: quickactions
- redis_slot: quickactions
aggregation: weekly
- name: i_quickactions_add_contacts
- category: quickactions
- redis_slot: quickactions
aggregation: weekly
- name: i_quickactions_remove_contacts
- category: quickactions
- redis_slot: quickactions
+ aggregation: weekly
+- name: i_quickactions_type
+ aggregation: weekly
+- name: i_quickactions_blocked_by
+ aggregation: weekly
+- name: i_quickactions_blocks
aggregation: weekly
diff --git a/lib/gitlab/usage_data_counters/known_events/work_items.yml b/lib/gitlab/usage_data_counters/known_events/work_items.yml
index d088b6d7e5a..a6e5b9e1af5 100644
--- a/lib/gitlab/usage_data_counters/known_events/work_items.yml
+++ b/lib/gitlab/usage_data_counters/known_events/work_items.yml
@@ -1,42 +1,21 @@
---
- name: users_updating_work_item_title
- category: work_items
- redis_slot: users
aggregation: weekly
- feature_flag: track_work_items_activity
- name: users_creating_work_items
- category: work_items
- redis_slot: users
aggregation: weekly
- feature_flag: track_work_items_activity
- name: users_updating_work_item_dates
- category: work_items
- redis_slot: users
aggregation: weekly
- feature_flag: track_work_items_activity
- name: users_updating_work_item_labels
- category: work_items
- redis_slot: users
aggregation: weekly
- feature_flag: track_work_items_activity
- name: users_updating_work_item_milestone
- category: work_items
- redis_slot: users
aggregation: weekly
- feature_flag: track_work_items_activity
- name: users_updating_work_item_iteration
# The event tracks an EE feature.
# It's added here so it can be aggregated into the CE/EE 'OR' aggregate metrics.
# It will report 0 for CE instances and should not be used with 'AND' aggregators.
- category: work_items
- redis_slot: users
aggregation: weekly
- feature_flag: track_work_items_activity
- name: users_updating_weight_estimate
# The event tracks an EE feature.
# It's added here so it can be aggregated into the CE/EE 'OR' aggregate metrics.
# It will report 0 for CE instances and should not be used with 'AND' aggregators.
- category: work_items
- redis_slot: users
aggregation: weekly
- feature_flag: track_work_items_activity
diff --git a/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb
index c8768164710..fceeacb60ca 100644
--- a/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb
+++ b/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb
@@ -68,8 +68,6 @@ module Gitlab
track_unique_action_by_merge_request(MR_CREATE_ACTION, merge_request)
project = merge_request.target_project
- return unless Feature.enabled?(:route_hll_to_snowplow_phase2, project.namespace)
-
Gitlab::Tracking.event(
name,
:create,
@@ -99,8 +97,6 @@ module Gitlab
track_unique_action_by_user(MR_APPROVE_ACTION, user)
project = merge_request.target_project
- return unless Feature.enabled?(:route_hll_to_snowplow_phase2, project.namespace)
-
Gitlab::Tracking.event(
name,
:approve,
diff --git a/lib/gitlab/usage_data_counters/track_unique_events.rb b/lib/gitlab/usage_data_counters/track_unique_events.rb
deleted file mode 100644
index 20da9665876..00000000000
--- a/lib/gitlab/usage_data_counters/track_unique_events.rb
+++ /dev/null
@@ -1,81 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module UsageDataCounters
- module TrackUniqueEvents
- WIKI_ACTION = :wiki_action
- DESIGN_ACTION = :design_action
- PUSH_ACTION = :project_action
- MERGE_REQUEST_ACTION = :merge_request_action
-
- GIT_WRITE_ACTIONS = [WIKI_ACTION, DESIGN_ACTION, PUSH_ACTION].freeze
- GIT_WRITE_ACTION = :git_write_action
-
- ACTION_TRANSFORMATIONS = HashWithIndifferentAccess.new({
- wiki: {
- created: WIKI_ACTION,
- updated: WIKI_ACTION,
- destroyed: WIKI_ACTION
- },
- design: {
- created: DESIGN_ACTION,
- updated: DESIGN_ACTION,
- destroyed: DESIGN_ACTION
- },
- project: {
- pushed: PUSH_ACTION
- },
- merge_request: {
- closed: MERGE_REQUEST_ACTION,
- merged: MERGE_REQUEST_ACTION,
- created: MERGE_REQUEST_ACTION,
- commented: MERGE_REQUEST_ACTION
- }
- }).freeze
-
- class << self
- def track_event(event_action:, event_target:, author_id:, time: Time.zone.now)
- return unless valid_target?(event_target)
- return unless valid_action?(event_action)
-
- transformed_target = transform_target(event_target)
- transformed_action = transform_action(event_action, transformed_target)
-
- return unless Gitlab::UsageDataCounters::HLLRedisCounter.known_event?(transformed_action.to_s)
-
- Gitlab::UsageDataCounters::HLLRedisCounter.track_event(transformed_action.to_s, values: author_id, time: time)
-
- track_git_write_action(author_id, transformed_action, time)
- end
-
- def count_unique_events(event_action:, date_from:, date_to:)
- Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: event_action.to_s, start_date: date_from, end_date: date_to)
- end
-
- private
-
- def transform_action(event_action, event_target)
- ACTION_TRANSFORMATIONS.dig(event_target, event_action) || event_action
- end
-
- def transform_target(event_target)
- Event::TARGET_TYPES.key(event_target)
- end
-
- def valid_target?(target)
- Event::TARGET_TYPES.value?(target)
- end
-
- def valid_action?(action)
- Event.actions.key?(action)
- end
-
- def track_git_write_action(author_id, transformed_action, time)
- return unless GIT_WRITE_ACTIONS.include?(transformed_action)
-
- Gitlab::UsageDataCounters::HLLRedisCounter.track_event(GIT_WRITE_ACTION, values: author_id, time: time)
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/usage_data_counters/work_item_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/work_item_activity_unique_counter.rb
index b99c9ebb24f..9de575d8567 100644
--- a/lib/gitlab/usage_data_counters/work_item_activity_unique_counter.rb
+++ b/lib/gitlab/usage_data_counters/work_item_activity_unique_counter.rb
@@ -33,7 +33,7 @@ module Gitlab
private
def track_unique_action(action, author)
- return unless author
+ return unless author && Feature.enabled?(:track_work_items_activity)
Gitlab::UsageDataCounters::HLLRedisCounter.track_event(action, values: author.id)
end
diff --git a/lib/gitlab/usage_data_metrics.rb b/lib/gitlab/usage_data_metrics.rb
index 48f695d5db1..8948d621d3c 100644
--- a/lib/gitlab/usage_data_metrics.rb
+++ b/lib/gitlab/usage_data_metrics.rb
@@ -8,10 +8,6 @@ module Gitlab
build_payload(:with_value)
end
- def suggested_names
- build_payload(:with_suggested_name)
- end
-
private
def build_payload(method_symbol)
diff --git a/lib/gitlab/usage_data_non_sql_metrics.rb b/lib/gitlab/usage_data_non_sql_metrics.rb
index 79d4b45a1ce..71386a58ba7 100644
--- a/lib/gitlab/usage_data_non_sql_metrics.rb
+++ b/lib/gitlab/usage_data_non_sql_metrics.rb
@@ -40,13 +40,6 @@ module Gitlab
def minimum_id(model, column = nil)
end
-
- def jira_integration_data
- {
- projects_jira_server_active: 0,
- projects_jira_cloud_active: 0
- }
- end
end
end
end
diff --git a/lib/gitlab/usage_data_queries.rb b/lib/gitlab/usage_data_queries.rb
index 3a163e5dde9..534a08cad9a 100644
--- a/lib/gitlab/usage_data_queries.rb
+++ b/lib/gitlab/usage_data_queries.rb
@@ -68,13 +68,6 @@ module Gitlab
end
end
- def jira_integration_data
- {
- projects_jira_server_active: 0,
- projects_jira_cloud_active: 0
- }
- end
-
def topology_usage_data
{
duration_s: 0,
diff --git a/lib/gitlab/utils/email.rb b/lib/gitlab/utils/email.rb
index c65d7165263..5eb57a66c63 100644
--- a/lib/gitlab/utils/email.rb
+++ b/lib/gitlab/utils/email.rb
@@ -8,19 +8,81 @@ module Gitlab
# Replaces most visible characters with * to obfuscate an email address
# deform adds a fix number of * to ensure the address cannot be guessed. Also obfuscates TLD with **
def obfuscated_email(email, deform: false)
- regex = ::Gitlab::UntrustedRegexp.new('^(..?)(.*)(@.?)(.*)(\..+)$')
- match = regex.match(email)
- return email unless match
-
- if deform
- # Ensure we can show two characters for the username, even if the username has
- # only one character. Boring solution is to just duplicate the character.
- email_start = match[1]
- email_start += email_start if email_start.length == 1
-
- email_start + '*' * 5 + match[3] + '*' * 5 + "#{match[5][0..1]}**"
- else
- match[1] + '*' * (match[2] || '').length + match[3] + '*' * (match[4] || '').length + match[5]
+ return email if email.empty?
+
+ masker_class = deform ? Deform : Symmetrical
+ masker_class.new(email).masked
+ end
+
+ class Masker
+ attr_reader :local_part, :sub_domain, :toplevel_domain, :at, :dot
+
+ def initialize(original)
+ @original = original
+ @local_part, @at, domain = original.rpartition('@')
+ @sub_domain, @dot, @toplevel_domain = domain.rpartition('.')
+
+ @at = nil if @at.empty?
+ @dot = nil if @dot.empty?
+ end
+
+ def masked
+ masked = [
+ local_part,
+ at,
+ sub_domain,
+ dot,
+ toplevel_domain
+ ].compact.join('')
+
+ masked = mask(@original, visible_length: 1) if masked == @original
+
+ masked
+ end
+
+ private
+
+ def mask(plain, visible_length:, star_length: nil)
+ return if plain.empty?
+ return plain if visible_length < 0
+
+ plain = enlarge_if_needed(plain, visible_length)
+
+ star_length = plain.length - visible_length if star_length.nil?
+
+ first = plain[0, visible_length]
+ stars = '*' * star_length
+
+ "#{first}#{stars}"
+ end
+
+ def enlarge_if_needed(string, min)
+ string.ljust(min, string.first)
+ end
+ end
+
+ class Symmetrical < Masker
+ def local_part
+ mask(@local_part, visible_length: 2)
+ end
+
+ def sub_domain
+ mask(@sub_domain, visible_length: 1)
+ end
+ end
+
+ # Implements https://design.gitlab.com/usability/obfuscation#email-addresses
+ class Deform < Masker
+ def local_part
+ mask(@local_part, visible_length: 2, star_length: 5)
+ end
+
+ def sub_domain
+ mask(@sub_domain, visible_length: 1, star_length: 5)
+ end
+
+ def toplevel_domain
+ mask(@toplevel_domain, visible_length: 1, star_length: 2)
end
end
end
diff --git a/lib/gitlab/utils/error_message.rb b/lib/gitlab/utils/error_message.rb
new file mode 100644
index 00000000000..cd9af95bfc3
--- /dev/null
+++ b/lib/gitlab/utils/error_message.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Utils
+ module ErrorMessage
+ extend self
+
+ UF_ERROR_PREFIX = 'UF'
+
+ def to_user_facing(message)
+ prefixed_error_message(message, UF_ERROR_PREFIX)
+ end
+
+ def prefixed_error_message(message, prefix)
+ "#{prefix} #{message}"
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/utils/override.rb b/lib/gitlab/utils/override.rb
index 7f43e25e50d..1d02bcbb2d2 100644
--- a/lib/gitlab/utils/override.rb
+++ b/lib/gitlab/utils/override.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
-require_dependency 'gitlab/utils'
-require_dependency 'gitlab/environment'
+require_relative '../utils'
+require_relative '../environment'
module Gitlab
module Utils
diff --git a/lib/gitlab/utils/strong_memoize.rb b/lib/gitlab/utils/strong_memoize.rb
index eb44b7ddd95..2b3841b8f09 100644
--- a/lib/gitlab/utils/strong_memoize.rb
+++ b/lib/gitlab/utils/strong_memoize.rb
@@ -35,6 +35,27 @@ module Gitlab
end
end
+ # Works the same way as "strong_memoize" but takes
+ # a second argument - expire_in. This allows invalidate
+ # the data after specified number of seconds
+ def strong_memoize_with_expiration(name, expire_in)
+ key = ivar(name)
+ expiration_key = "#{key}_expired_at"
+
+ if instance_variable_defined?(expiration_key)
+ expire_at = instance_variable_get(expiration_key)
+ clear_memoization(name) if Time.current > expire_at
+ end
+
+ if instance_variable_defined?(key)
+ instance_variable_get(key)
+ else
+ value = instance_variable_set(key, yield)
+ instance_variable_set(expiration_key, Time.current + expire_in)
+ value
+ end
+ end
+
def strong_memoize_with(name, *args)
container = strong_memoize(name) { {} }
diff --git a/lib/gitlab/utils/uniquify.rb b/lib/gitlab/utils/uniquify.rb
new file mode 100644
index 00000000000..b5908d18103
--- /dev/null
+++ b/lib/gitlab/utils/uniquify.rb
@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+# Uniquify
+#
+# Return a version of the given 'base' string that is unique
+# by appending a counter to it. Uniqueness is determined by
+# repeated calls to the passed block.
+#
+# You can pass an initial value for the counter, if not given
+# counting starts from 1.
+#
+# If `base` is a function/proc, we expect that calling it with a
+# candidate counter returns a string to test/return.
+
+module Gitlab
+ module Utils
+ class Uniquify
+ def initialize(counter = nil)
+ @counter = counter
+ end
+
+ def string(base)
+ @base = base
+
+ increment_counter! while yield(base_string)
+ base_string
+ end
+
+ private
+
+ def base_string
+ if @base.respond_to?(:call)
+ @base.call(@counter)
+ else
+ "#{@base}#{@counter}"
+ end
+ end
+
+ def increment_counter!
+ @counter ||= 0
+ @counter += 1
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/utils/usage_data.rb b/lib/gitlab/utils/usage_data.rb
index fab8617bcda..4106084b301 100644
--- a/lib/gitlab/utils/usage_data.rb
+++ b/lib/gitlab/utils/usage_data.rb
@@ -255,33 +255,6 @@ module Gitlab
end
end
- # rubocop: disable UsageData/LargeTable:
- def jira_integration_data
- with_metadata do
- data = {
- projects_jira_server_active: 0,
- projects_jira_cloud_active: 0
- }
-
- # rubocop: disable CodeReuse/ActiveRecord
- ::Integrations::Jira.active.includes(:jira_tracker_data).find_in_batches(batch_size: 100) do |services|
- counts = services.group_by do |service|
- # TODO: Simplify as part of https://gitlab.com/gitlab-org/gitlab/issues/29404
- service_url = service.data_fields&.url || (service.properties && service.properties['url'])
- service_url&.include?('.atlassian.net') ? :cloud : :server
- end
-
- data[:projects_jira_server_active] += counts[:server].size if counts[:server]
- data[:projects_jira_cloud_active] += counts[:cloud].size if counts[:cloud]
- end
-
- data
- end
- end
-
- # rubocop: enable CodeReuse/ActiveRecord
- # rubocop: enable UsageData/LargeTable:
-
def minimum_id(model, column = nil)
key = :"#{model.name.downcase.gsub('::', '_')}_minimum_id"
column_to_read = column || :id
diff --git a/lib/gitlab/utils/username_and_email_generator.rb b/lib/gitlab/utils/username_and_email_generator.rb
new file mode 100644
index 00000000000..38c9bb7050d
--- /dev/null
+++ b/lib/gitlab/utils/username_and_email_generator.rb
@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require 'securerandom'
+
+module Gitlab
+ module Utils
+ class UsernameAndEmailGenerator
+ include Gitlab::Utils::StrongMemoize
+
+ def initialize(username_prefix:, email_domain: Gitlab.config.gitlab.host)
+ @username_prefix = username_prefix
+ @email_domain = email_domain
+ end
+
+ def username
+ uniquify.string(->(counter) { Kernel.sprintf(username_pattern, counter) }) do |suggested_username|
+ ::Namespace.by_path(suggested_username) || ::User.find_by_any_email(email_for(suggested_username))
+ end
+ end
+ strong_memoize_attr :username
+
+ def email
+ email_for(username)
+ end
+ strong_memoize_attr :email
+
+ private
+
+ def username_pattern
+ "#{@username_prefix}_#{SecureRandom.hex(16)}%s"
+ end
+
+ def email_for(name)
+ "#{name}@#{@email_domain}"
+ end
+
+ def uniquify
+ Gitlab::Utils::Uniquify.new
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/verify/batch_verifier.rb b/lib/gitlab/verify/batch_verifier.rb
index 71d106db742..bff95743dbd 100644
--- a/lib/gitlab/verify/batch_verifier.rb
+++ b/lib/gitlab/verify/batch_verifier.rb
@@ -34,7 +34,7 @@ module Gitlab
private
def run_batch_for(batch)
- batch.map { |upload| verify(upload) }.compact.to_h
+ batch.filter_map { |upload| verify(upload) }.to_h
end
def verify(object)
diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb
index 02418c45e73..79131404465 100644
--- a/lib/gitlab/workhorse.rb
+++ b/lib/gitlab/workhorse.rb
@@ -156,11 +156,14 @@ module Gitlab
]
end
- def send_url(url, allow_redirects: false)
+ def send_url(url, allow_redirects: false, method: 'GET', body: nil, headers: nil)
params = {
'URL' => url,
- 'AllowRedirects' => allow_redirects
- }
+ 'AllowRedirects' => allow_redirects,
+ 'Body' => body.to_s,
+ 'Header' => headers,
+ 'Method' => method
+ }.compact
[
SEND_DATA_HEADER,
diff --git a/lib/gitlab/x509/signature.rb b/lib/gitlab/x509/signature.rb
index d6bbb8bb2cb..d101a6d2522 100644
--- a/lib/gitlab/x509/signature.rb
+++ b/lib/gitlab/x509/signature.rb
@@ -110,8 +110,6 @@ module Gitlab
else
false
end
- else
- nil
end
rescue StandardError
nil
diff --git a/lib/gitlab_settings.rb b/lib/gitlab_settings.rb
new file mode 100644
index 00000000000..7f37a125332
--- /dev/null
+++ b/lib/gitlab_settings.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require 'active_support'
+require 'active_support/core_ext/hash'
+
+require_relative 'gitlab_settings/settings'
+require_relative 'gitlab_settings/options'
+
+module GitlabSettings
+ MissingSetting = Class.new(StandardError)
+
+ def self.load(source = nil, section = nil, &block)
+ ::GitlabSettings::Settings
+ .new(source, section)
+ .extend(Module.new(&block))
+ end
+end
diff --git a/lib/gitlab_settings/options.rb b/lib/gitlab_settings/options.rb
new file mode 100644
index 00000000000..5bdac74c2f6
--- /dev/null
+++ b/lib/gitlab_settings/options.rb
@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module GitlabSettings
+ class Options
+ # Recursively build GitlabSettings::Options
+ def self.build(obj)
+ case obj
+ when Hash
+ new(obj.transform_values { |value| build(value) })
+ when Array
+ obj.map { |value| build(value) }
+ else
+ obj
+ end
+ end
+
+ def initialize(value)
+ @options = value.deep_stringify_keys
+ end
+
+ def [](key)
+ @options[key.to_s]
+ end
+
+ def []=(key, value)
+ @options[key.to_s] = self.class.build(value)
+ end
+
+ def key?(name)
+ @options.key?(name.to_s) || @options.key?(name.to_sym)
+ end
+ alias_method :has_key?, :key?
+
+ def to_hash
+ @options.deep_transform_values do |option|
+ case option
+ when self.class
+ option.to_hash
+ else
+ option
+ end
+ end
+ end
+ alias_method :to_h, :to_hash
+
+ def merge(other)
+ self.class.build(to_hash.merge(other.deep_stringify_keys))
+ end
+
+ def deep_merge(other)
+ self.class.build(to_hash.deep_merge(other.deep_stringify_keys))
+ end
+
+ def is_a?(klass)
+ return true if klass == Hash
+
+ super(klass)
+ end
+
+ def method_missing(name, *args, &block)
+ name_string = +name.to_s
+
+ if name_string.chomp!("=")
+ return self[name_string] = args.first if key?(name_string)
+ elsif key?(name_string)
+ return self[name_string]
+ end
+
+ return @options.public_send(name, *args, &block) if @options.respond_to?(name) # rubocop: disable GitlabSecurity/PublicSend
+
+ raise ::GitlabSettings::MissingSetting, "option '#{name}' not defined"
+ end
+
+ def respond_to_missing?(name, include_all = false)
+ return true if key?(name)
+
+ @options.respond_to?(name, include_all)
+ end
+ end
+end
diff --git a/lib/gitlab_settings/settings.rb b/lib/gitlab_settings/settings.rb
new file mode 100644
index 00000000000..ae2ed4eac05
--- /dev/null
+++ b/lib/gitlab_settings/settings.rb
@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module GitlabSettings
+ class Settings
+ attr_reader :source
+
+ def initialize(source, section)
+ raise(ArgumentError, 'config source is required') if source.blank?
+ raise(ArgumentError, 'config section is required') if section.blank?
+
+ @source = source
+ @section = section
+ @loaded = false
+ end
+
+ def reload!
+ yaml = ActiveSupport::ConfigurationFile.parse(source)
+ all_configs = yaml.deep_stringify_keys
+ configs = all_configs[section]
+
+ @config = Options.build(configs).tap do
+ @loaded = true
+ end
+ end
+
+ def method_missing(name, *args)
+ reload! unless @loaded
+
+ config.public_send(name, *args) # rubocop: disable GitlabSecurity/PublicSend
+ end
+
+ def respond_to_missing?(name, include_all = false)
+ config.respond_to?(name, include_all)
+ end
+
+ private
+
+ attr_reader :config, :section
+ end
+end
diff --git a/lib/object_storage/config.rb b/lib/object_storage/config.rb
index 056e22278dd..8a2044a8cba 100644
--- a/lib/object_storage/config.rb
+++ b/lib/object_storage/config.rb
@@ -6,6 +6,18 @@ module ObjectStorage
AZURE_PROVIDER = 'AzureRM'
GOOGLE_PROVIDER = 'Google'
+ LOCATIONS = {
+ artifacts: Gitlab.config.artifacts,
+ ci_secure_files: Gitlab.config.ci_secure_files,
+ dependency_proxy: Gitlab.config.dependency_proxy,
+ external_diffs: Gitlab.config.external_diffs,
+ lfs: Gitlab.config.lfs,
+ packages: Gitlab.config.packages,
+ pages: Gitlab.config.pages,
+ terraform_state: Gitlab.config.terraform_state,
+ uploads: Gitlab.config.uploads
+ }.freeze
+
attr_reader :options
def initialize(options)
@@ -13,7 +25,7 @@ module ObjectStorage
end
def credentials
- @credentials ||= options[:connection] || {}
+ @credentials ||= connection_params
end
def storage_options
@@ -86,6 +98,16 @@ module ObjectStorage
private
+ def connection_params
+ base_params = options[:connection] || {}
+
+ return base_params unless base_params[:provider].to_s == AWS_PROVIDER
+ return base_params unless ::Gitlab::FIPS.enabled?
+
+ # In fog-aws, this disables the use of Content-Md5: https://github.com/fog/fog-aws/pull/668
+ base_params.merge({ disable_content_md5_validation: true })
+ end
+
# This returns a Hash of HTTP encryption headers to send along to S3.
#
# They can also be passed in as Fog::AWS::Storage::File attributes, since there
diff --git a/lib/object_storage/direct_upload.rb b/lib/object_storage/direct_upload.rb
index 7d2f825e119..588e3c3e89f 100644
--- a/lib/object_storage/direct_upload.rb
+++ b/lib/object_storage/direct_upload.rb
@@ -23,9 +23,9 @@ module ObjectStorage
MINIMUM_MULTIPART_SIZE = 5.megabytes
attr_reader :config, :credentials, :bucket_name, :object_name
- attr_reader :has_length, :maximum_size
+ attr_reader :has_length, :maximum_size, :skip_delete
- def initialize(config, object_name, has_length:, maximum_size: nil)
+ def initialize(config, object_name, has_length:, maximum_size: nil, skip_delete: false)
unless has_length
raise ArgumentError, 'maximum_size has to be specified if length is unknown' unless maximum_size
end
@@ -36,6 +36,7 @@ module ObjectStorage
@object_name = object_name
@has_length = has_length
@maximum_size = maximum_size
+ @skip_delete = skip_delete
end
def to_hash
@@ -44,7 +45,7 @@ module ObjectStorage
GetURL: get_url,
StoreURL: store_url,
DeleteURL: delete_url,
- SkipDelete: false,
+ SkipDelete: skip_delete,
MultipartUpload: multipart_upload_hash,
CustomPutHeaders: true,
PutHeaders: upload_options
@@ -260,7 +261,7 @@ module ObjectStorage
end
def connection
- @connection ||= ::Fog::Storage.new(credentials)
+ @connection ||= ::Fog::Storage.new(credentials.to_hash)
end
end
end
diff --git a/lib/object_storage/pending_direct_upload.rb b/lib/object_storage/pending_direct_upload.rb
new file mode 100644
index 00000000000..3e84bc4ebc9
--- /dev/null
+++ b/lib/object_storage/pending_direct_upload.rb
@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module ObjectStorage
+ class PendingDirectUpload
+ KEY = 'pending_direct_uploads'
+
+ def self.prepare(location_identifier, path)
+ ::Gitlab::Redis::SharedState.with do |redis|
+ # We need to store the location_identifier together with the timestamp to properly delete
+ # this object if ever this upload gets stale. The location identifier will be used
+ # by the clean up worker to properly generate the storage options through ObjectStorage::Config.for_location
+ redis.hset(KEY, key(location_identifier, path), Time.current.utc.to_i)
+ end
+ end
+
+ def self.exists?(location_identifier, path)
+ ::Gitlab::Redis::SharedState.with do |redis|
+ redis.hexists(KEY, key(location_identifier, path))
+ end
+ end
+
+ def self.complete(location_identifier, path)
+ ::Gitlab::Redis::SharedState.with do |redis|
+ redis.hdel(KEY, key(location_identifier, path))
+ end
+ end
+
+ def self.key(location_identifier, path)
+ [location_identifier, path].join(':')
+ end
+ end
+end
diff --git a/lib/peek/views/zoekt.rb b/lib/peek/views/zoekt.rb
new file mode 100644
index 00000000000..d1e9a6d68ed
--- /dev/null
+++ b/lib/peek/views/zoekt.rb
@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Peek
+ module Views
+ class Zoekt < DetailedView
+ DEFAULT_THRESHOLDS = {
+ calls: 3,
+ duration: 500,
+ individual_call: 500
+ }.freeze
+
+ THRESHOLDS = {
+ production: {
+ calls: 5,
+ duration: 1000,
+ individual_call: 1000
+ }
+ }.freeze
+
+ def key
+ 'zkt'
+ end
+
+ def self.thresholds
+ @thresholds ||= THRESHOLDS.fetch(Rails.env.to_sym, DEFAULT_THRESHOLDS)
+ end
+
+ private
+
+ def duration
+ ::Gitlab::Instrumentation::Zoekt.query_time * 1000
+ end
+
+ def calls
+ ::Gitlab::Instrumentation::Zoekt.get_request_count
+ end
+
+ def call_details
+ ::Gitlab::Instrumentation::Zoekt.detail_store
+ end
+
+ def format_call_details(call)
+ super.merge(request: "#{call[:method]} #{call[:path]}?#{(call[:params] || {}).to_query}")
+ end
+ end
+ end
+end
diff --git a/lib/product_analytics/settings.rb b/lib/product_analytics/settings.rb
new file mode 100644
index 00000000000..5d52965f5be
--- /dev/null
+++ b/lib/product_analytics/settings.rb
@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module ProductAnalytics
+ class Settings
+ CONFIG_KEYS = (%w[jitsu_host jitsu_project_xid jitsu_administrator_email jitsu_administrator_password] +
+ %w[product_analytics_data_collector_host product_analytics_clickhouse_connection_string] +
+ %w[cube_api_base_url cube_api_key]).freeze
+
+ def initialize(project:)
+ @project = project
+ end
+
+ def enabled?
+ ::Gitlab::CurrentSettings.product_analytics_enabled? && configured?
+ end
+
+ # rubocop:disable GitlabSecurity/PublicSend
+ def configured?
+ CONFIG_KEYS.all? do |key|
+ @project.project_setting.public_send(key).present? ||
+ ::Gitlab::CurrentSettings.public_send(key).present?
+ end
+ end
+
+ CONFIG_KEYS.each do |key|
+ define_method key.to_sym do
+ @project.project_setting.public_send(key).presence || ::Gitlab::CurrentSettings.public_send(key)
+ end
+ end
+ # rubocop:enable GitlabSecurity/PublicSend
+
+ class << self
+ def for_project(project)
+ ProductAnalytics::Settings.new(project: project)
+ end
+ end
+ end
+end
diff --git a/lib/product_analytics/tracker.rb b/lib/product_analytics/tracker.rb
deleted file mode 100644
index e5550c2e6ef..00000000000
--- a/lib/product_analytics/tracker.rb
+++ /dev/null
@@ -1,11 +0,0 @@
-# frozen_string_literal: true
-
-module ProductAnalytics
- class Tracker
- # The file is located in the /public directory
- URL = Gitlab.config.gitlab.url + '/-/sp.js'
-
- # The collector URL minus protocol and /i
- COLLECTOR_URL = Gitlab.config.gitlab.url.sub(%r{\Ahttps?\://}, '') + '/-/collector'
- end
-end
diff --git a/lib/safe_zip/extract.rb b/lib/safe_zip/extract.rb
index b86941e6bea..3403e3e429e 100644
--- a/lib/safe_zip/extract.rb
+++ b/lib/safe_zip/extract.rb
@@ -1,6 +1,13 @@
# frozen_string_literal: true
module SafeZip
+ # SafeZip::Extract provides a safe interface
+ # to extract specific directories or files within a `zip` archive.
+ #
+ # @example Extract directories to destination
+ # SafeZip::Extract.new(archive_file).extract(directories: ['app/', 'test/'], to: destination_path)
+ # @example Extract files to destination
+ # SafeZip::Extract.new(archive_file).extract(files: ['index.html', 'app/index.js'], to: destination_path)
class Extract
Error = Class.new(StandardError)
PermissionDeniedError = Class.new(Error)
@@ -17,6 +24,20 @@ module SafeZip
@archive_path = archive_file
end
+ # extract given files or directories from the archive into the destination path
+ #
+ # @param [Hash] opts the options for extraction.
+ # @option opts [Array<String] :files list of files to be extracted
+ # @option opts [Array<String] :directories list of directories to be extracted
+ # @option opts [String] :to destination path
+ #
+ # @raise [PermissionDeniedError]
+ # @raise [SymlinkSourceDoesNotExistError]
+ # @raise [UnsupportedEntryError]
+ # @raise [EntrySizeError]
+ # @raise [AlreadyExistsError]
+ # @raise [NoMatchingError]
+ # @raise [ExtractError]
def extract(opts = {})
params = SafeZip::ExtractParams.new(**opts)
diff --git a/lib/sidebars/admin/base_menu.rb b/lib/sidebars/admin/base_menu.rb
new file mode 100644
index 00000000000..897a193f672
--- /dev/null
+++ b/lib/sidebars/admin/base_menu.rb
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module Admin
+ class BaseMenu < ::Sidebars::Menu
+ override :render?
+ def render?
+ return false unless context.current_user
+
+ context.current_user.can_admin_all_resources?
+ end
+ end
+ end
+end
diff --git a/lib/sidebars/admin/menus/abuse_reports_menu.rb b/lib/sidebars/admin/menus/abuse_reports_menu.rb
new file mode 100644
index 00000000000..72f4d6e6590
--- /dev/null
+++ b/lib/sidebars/admin/menus/abuse_reports_menu.rb
@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module Admin
+ module Menus
+ class AbuseReportsMenu < ::Sidebars::Admin::BaseMenu
+ override :link
+ def link
+ admin_abuse_reports_path
+ end
+
+ override :title
+ def title
+ s_('Admin|Abuse Reports')
+ end
+
+ override :sprite_icon
+ def sprite_icon
+ 'slight-frown'
+ end
+
+ override :has_pill?
+ def has_pill?
+ true
+ end
+
+ override :pill_count
+ def pill_count
+ @pill_count ||= AbuseReport.count(:all)
+ end
+
+ override :active_routes
+ def active_routes
+ { controller: :abuse_reports }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/sidebars/admin/menus/admin_overview_menu.rb b/lib/sidebars/admin/menus/admin_overview_menu.rb
new file mode 100644
index 00000000000..57c9ff4dcb0
--- /dev/null
+++ b/lib/sidebars/admin/menus/admin_overview_menu.rb
@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module Admin
+ module Menus
+ class AdminOverviewMenu < ::Sidebars::Admin::BaseMenu
+ override :configure_menu_items
+ def configure_menu_items
+ add_item(dashboard_menu_item)
+ add_item(projects_menu_item)
+ add_item(users_menu_item)
+ add_item(groups_menu_item)
+ add_item(topics_menu_item)
+ add_item(gitaly_servers_menu_item)
+
+ true
+ end
+
+ override :title
+ def title
+ s_('Admin|Overview')
+ end
+
+ override :sprite_icon
+ def sprite_icon
+ 'overview'
+ end
+
+ override :extra_container_html_options
+ def extra_container_html_options
+ { 'data-qa-selector': 'admin_overview_submenu_content' }
+ end
+
+ private
+
+ def dashboard_menu_item
+ ::Sidebars::MenuItem.new(
+ title: _('Dashboard'),
+ link: admin_root_path,
+ active_routes: { controller: 'dashboard' },
+ item_id: :dashboard
+ )
+ end
+
+ def projects_menu_item
+ ::Sidebars::MenuItem.new(
+ title: _('Projects'),
+ link: admin_projects_path,
+ active_routes: { controller: 'admin/projects' },
+ item_id: :projects
+ )
+ end
+
+ def users_menu_item
+ ::Sidebars::MenuItem.new(
+ title: _('Users'),
+ link: admin_users_path,
+ active_routes: { controller: 'users' },
+ item_id: :users,
+ container_html_options: { 'data-qa-selector': 'admin_overview_users_link' }
+ )
+ end
+
+ def groups_menu_item
+ ::Sidebars::MenuItem.new(
+ title: _('Groups'),
+ link: admin_groups_path,
+ active_routes: { controller: 'groups' },
+ item_id: :groups,
+ container_html_options: { 'data-qa-selector': 'admin_overview_groups_link' }
+ )
+ end
+
+ def topics_menu_item
+ ::Sidebars::MenuItem.new(
+ title: _('Topics'),
+ link: admin_topics_path,
+ active_routes: { controller: 'admin/topics' },
+ item_id: :topics
+ )
+ end
+
+ def gitaly_servers_menu_item
+ ::Sidebars::MenuItem.new(
+ title: _('Gitaly Servers'),
+ link: admin_gitaly_servers_path,
+ active_routes: { controller: 'gitaly_servers' },
+ item_id: :gitaly_servers
+ )
+ end
+ end
+ end
+ end
+end
diff --git a/lib/sidebars/admin/menus/admin_settings_menu.rb b/lib/sidebars/admin/menus/admin_settings_menu.rb
new file mode 100644
index 00000000000..4656e0f33e2
--- /dev/null
+++ b/lib/sidebars/admin/menus/admin_settings_menu.rb
@@ -0,0 +1,151 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module Admin
+ module Menus
+ class AdminSettingsMenu < ::Sidebars::Admin::BaseMenu
+ override :configure_menu_items
+ def configure_menu_items
+ add_item(general_settings_menu_item)
+ add_item(integrations_menu_item)
+ add_item(repository_menu_item)
+ add_item(ci_cd_menu_item)
+ add_item(reporting_menu_item)
+ add_item(metrics_and_profiling_menu_item)
+ add_item(service_usage_data_menu_item)
+ add_item(network_settings_menu_item)
+ add_item(appearance_menu_item)
+ add_item(preferences_menu_item)
+
+ true
+ end
+
+ override :title
+ def title
+ s_('Admin|Settings')
+ end
+
+ override :sprite_icon
+ def sprite_icon
+ 'settings'
+ end
+
+ override :extra_container_html_options
+ def extra_container_html_options
+ { 'data-qa-selector': 'admin_settings_menu_link' }
+ end
+
+ override :separated?
+ def separated?
+ true
+ end
+
+ private
+
+ def general_settings_menu_item
+ ::Sidebars::MenuItem.new(
+ title: _('General'),
+ link: general_admin_application_settings_path,
+ active_routes: { path: 'admin/application_settings#general' },
+ item_id: :general_settings,
+ container_html_options: { 'data-qa-selector': 'admin_settings_general_link' }
+ )
+ end
+
+ def integrations_menu_item
+ return ::Sidebars::NilMenuItem.new(item_id: :admin_integrations) unless instance_level_integrations?
+
+ ::Sidebars::MenuItem.new(
+ title: _('Integrations'),
+ link: integrations_admin_application_settings_path,
+ active_routes: { path: %w[application_settings#integrations integrations#edit] },
+ item_id: :admin_integrations,
+ container_html_options: { 'data-qa-selector': 'admin_settings_integrations_link' }
+ )
+ end
+
+ def repository_menu_item
+ ::Sidebars::MenuItem.new(
+ title: _('Repository'),
+ link: repository_admin_application_settings_path,
+ active_routes: { path: 'admin/application_settings#repository' },
+ item_id: :admin_repository,
+ container_html_options: { 'data-qa-selector': 'admin_settings_repository_link' }
+ )
+ end
+
+ def ci_cd_menu_item
+ ::Sidebars::MenuItem.new(
+ title: _('CI/CD'),
+ link: ci_cd_admin_application_settings_path,
+ active_routes: { path: 'admin/application_settings#ci_cd' },
+ item_id: :admin_ci_cd
+ )
+ end
+
+ def reporting_menu_item
+ ::Sidebars::MenuItem.new(
+ title: _('Reporting'),
+ link: reporting_admin_application_settings_path,
+ active_routes: { path: 'admin/application_settings#reporting' },
+ item_id: :admin_reporting
+ )
+ end
+
+ def metrics_and_profiling_menu_item
+ ::Sidebars::MenuItem.new(
+ title: _('Metrics and profiling'),
+ link: metrics_and_profiling_admin_application_settings_path,
+ active_routes: { path: 'admin/application_settings#metrics_and_profiling' },
+ item_id: :admin_metrics,
+ container_html_options: { 'data-qa-selector': 'admin_settings_metrics_and_profiling_link' }
+ )
+ end
+
+ def service_usage_data_menu_item
+ ::Sidebars::MenuItem.new(
+ title: _('Service usage data'),
+ link: service_usage_data_admin_application_settings_path,
+ active_routes: { path: 'admin/application_settings#service_usage_data' },
+ item_id: :admin_service_usage
+ )
+ end
+
+ def network_settings_menu_item
+ ::Sidebars::MenuItem.new(
+ title: _('Network'),
+ link: network_admin_application_settings_path,
+ active_routes: { path: 'admin/application_settings#network' },
+ item_id: :admin_network,
+ container_html_options: { 'data-qa-selector': 'admin_settings_network_link' }
+ )
+ end
+
+ def appearance_menu_item
+ ::Sidebars::MenuItem.new(
+ title: _('Appearance'),
+ link: admin_application_settings_appearances_path,
+ active_routes: { path: 'admin/application_settings/appearances#show' },
+ item_id: :admin_appearance
+ )
+ end
+
+ def preferences_menu_item
+ ::Sidebars::MenuItem.new(
+ title: _('Preferences'),
+ link: preferences_admin_application_settings_path,
+ active_routes: { path: 'admin/application_settings#preferences' },
+ item_id: :admin_preferences,
+ container_html_options: { 'data-qa-selector': 'admin_settings_preferences_link' }
+ )
+ end
+
+ def instance_level_integrations?
+ !Gitlab.com?
+ end
+ end
+ end
+ end
+end
+
+Sidebars::Admin::Menus::AdminSettingsMenu.prepend_mod_with('Sidebars::Admin::Menus::AdminSettingsMenu')
diff --git a/lib/sidebars/admin/menus/analytics_menu.rb b/lib/sidebars/admin/menus/analytics_menu.rb
new file mode 100644
index 00000000000..944f7f6bba7
--- /dev/null
+++ b/lib/sidebars/admin/menus/analytics_menu.rb
@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module Admin
+ module Menus
+ class AnalyticsMenu < ::Sidebars::Admin::BaseMenu
+ override :configure_menu_items
+ def configure_menu_items
+ add_item(dev_ops_reports_menu_item)
+ add_item(usage_trends_menu_item)
+
+ true
+ end
+
+ override :title
+ def title
+ s_('Admin|Analytics')
+ end
+
+ override :sprite_icon
+ def sprite_icon
+ 'chart'
+ end
+
+ override :extra_container_html_options
+ def extra_container_html_options
+ { 'data-qa-selector': 'admin_sidebar_analytics_submenu_content' }
+ end
+
+ private
+
+ def dev_ops_reports_menu_item
+ ::Sidebars::MenuItem.new(
+ title: _('DevOps Reports'),
+ link: admin_dev_ops_reports_path,
+ active_routes: { controller: 'dev_ops_report' },
+ item_id: :dev_ops_reports,
+ container_html_options: { 'data-qa-selector': 'admin_analytics_link' }
+ )
+ end
+
+ def usage_trends_menu_item
+ ::Sidebars::MenuItem.new(
+ title: _('Usage Trends'),
+ link: admin_usage_trends_path,
+ active_routes: { controller: 'usage_trends' },
+ item_id: :usage_trends
+ )
+ end
+ end
+ end
+ end
+end
diff --git a/lib/sidebars/admin/menus/applications_menu.rb b/lib/sidebars/admin/menus/applications_menu.rb
new file mode 100644
index 00000000000..74116076735
--- /dev/null
+++ b/lib/sidebars/admin/menus/applications_menu.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module Admin
+ module Menus
+ class ApplicationsMenu < ::Sidebars::Admin::BaseMenu
+ override :link
+ def link
+ admin_applications_path
+ end
+
+ override :title
+ def title
+ s_('Admin|Applications')
+ end
+
+ override :sprite_icon
+ def sprite_icon
+ 'applications'
+ end
+
+ override :active_routes
+ def active_routes
+ { controller: :applications }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/sidebars/admin/menus/ci_cd_menu.rb b/lib/sidebars/admin/menus/ci_cd_menu.rb
new file mode 100644
index 00000000000..e6e8e77a448
--- /dev/null
+++ b/lib/sidebars/admin/menus/ci_cd_menu.rb
@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module Admin
+ module Menus
+ class CiCdMenu < ::Sidebars::Admin::BaseMenu
+ override :configure_menu_items
+ def configure_menu_items
+ add_item(runners_menu_item)
+ add_item(jobs_menu_item)
+
+ true
+ end
+
+ override :title
+ def title
+ s_('Admin|CI/CD')
+ end
+
+ override :sprite_icon
+ def sprite_icon
+ 'rocket'
+ end
+
+ private
+
+ def runners_menu_item
+ ::Sidebars::MenuItem.new(
+ title: _('Runners'),
+ link: admin_runners_path,
+ active_routes: { controller: 'runners' },
+ item_id: :runners
+ )
+ end
+
+ def jobs_menu_item
+ ::Sidebars::MenuItem.new(
+ title: _('Jobs'),
+ link: admin_jobs_path,
+ active_routes: { controller: 'jobs' },
+ item_id: :jobs
+ )
+ end
+ end
+ end
+ end
+end
diff --git a/lib/sidebars/admin/menus/deploy_keys_menu.rb b/lib/sidebars/admin/menus/deploy_keys_menu.rb
new file mode 100644
index 00000000000..4ffc6635f27
--- /dev/null
+++ b/lib/sidebars/admin/menus/deploy_keys_menu.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module Admin
+ module Menus
+ class DeployKeysMenu < ::Sidebars::Admin::BaseMenu
+ override :link
+ def link
+ admin_deploy_keys_path
+ end
+
+ override :title
+ def title
+ s_('Admin|Deploy Keys')
+ end
+
+ override :sprite_icon
+ def sprite_icon
+ 'key'
+ end
+
+ override :active_routes
+ def active_routes
+ { controller: :deploy_keys }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/sidebars/admin/menus/kubernetes_menu.rb b/lib/sidebars/admin/menus/kubernetes_menu.rb
new file mode 100644
index 00000000000..88b184290f1
--- /dev/null
+++ b/lib/sidebars/admin/menus/kubernetes_menu.rb
@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module Admin
+ module Menus
+ class KubernetesMenu < ::Sidebars::Admin::BaseMenu
+ override :link
+ def link
+ admin_clusters_path
+ end
+
+ override :title
+ def title
+ s_('Admin|Kubernetes')
+ end
+
+ override :sprite_icon
+ def sprite_icon
+ 'cloud-gear'
+ end
+
+ override :render?
+ def render?
+ current_user && current_user.can_admin_all_resources? && instance_clusters_enabled?
+ end
+
+ override :active_routes
+ def active_routes
+ { controller: :clusters }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/sidebars/admin/menus/labels_menu.rb b/lib/sidebars/admin/menus/labels_menu.rb
new file mode 100644
index 00000000000..32b4b53960a
--- /dev/null
+++ b/lib/sidebars/admin/menus/labels_menu.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module Admin
+ module Menus
+ class LabelsMenu < ::Sidebars::Admin::BaseMenu
+ override :link
+ def link
+ admin_labels_path
+ end
+
+ override :title
+ def title
+ s_('Admin|Labels')
+ end
+
+ override :sprite_icon
+ def sprite_icon
+ 'labels'
+ end
+
+ override :active_routes
+ def active_routes
+ { controller: :labels }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/sidebars/admin/menus/messages_menu.rb b/lib/sidebars/admin/menus/messages_menu.rb
new file mode 100644
index 00000000000..0d7110f42bf
--- /dev/null
+++ b/lib/sidebars/admin/menus/messages_menu.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module Admin
+ module Menus
+ class MessagesMenu < ::Sidebars::Admin::BaseMenu
+ override :link
+ def link
+ admin_broadcast_messages_path
+ end
+
+ override :title
+ def title
+ s_('Admin|Messages')
+ end
+
+ override :sprite_icon
+ def sprite_icon
+ 'messages'
+ end
+
+ override :active_routes
+ def active_routes
+ { controller: :broadcast_messages }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/sidebars/admin/menus/monitoring_menu.rb b/lib/sidebars/admin/menus/monitoring_menu.rb
new file mode 100644
index 00000000000..2da56e87144
--- /dev/null
+++ b/lib/sidebars/admin/menus/monitoring_menu.rb
@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module Admin
+ module Menus
+ class MonitoringMenu < ::Sidebars::Admin::BaseMenu
+ override :configure_menu_items
+ def configure_menu_items
+ add_item(system_info_menu_item)
+ add_item(background_migrations_menu_item)
+ add_item(background_jobs_menu_item)
+ add_item(health_check_menu_item)
+ true
+ end
+
+ override :title
+ def title
+ s_('Admin|Monitoring')
+ end
+
+ override :sprite_icon
+ def sprite_icon
+ 'monitor'
+ end
+
+ override :extra_container_html_options
+ def extra_container_html_options
+ { 'data-qa-selector': 'admin_monitoring_menu_link' }
+ end
+
+ private
+
+ def system_info_menu_item
+ ::Sidebars::MenuItem.new(
+ title: _('System Info'),
+ link: admin_system_info_path,
+ active_routes: { controller: 'system_info' },
+ item_id: :system_info
+ )
+ end
+
+ def background_migrations_menu_item
+ ::Sidebars::MenuItem.new(
+ title: _('Background Migrations'),
+ link: admin_background_migrations_path,
+ active_routes: { controller: 'background_migrations' },
+ item_id: :background_migrations
+ )
+ end
+
+ def background_jobs_menu_item
+ ::Sidebars::MenuItem.new(
+ title: _('Background Jobs'),
+ link: admin_background_jobs_path,
+ active_routes: { controller: 'background_jobs' },
+ item_id: :background_jobs
+ )
+ end
+
+ def health_check_menu_item
+ ::Sidebars::MenuItem.new(
+ title: _('Health Check'),
+ link: admin_health_check_path,
+ active_routes: { controller: 'health_check' },
+ item_id: :health_check
+ )
+ end
+ end
+ end
+ end
+end
+
+Sidebars::Admin::Menus::MonitoringMenu.prepend_mod_with('Sidebars::Admin::Menus::MonitoringMenu')
diff --git a/lib/sidebars/admin/menus/spam_logs_menu.rb b/lib/sidebars/admin/menus/spam_logs_menu.rb
new file mode 100644
index 00000000000..d01cd636e13
--- /dev/null
+++ b/lib/sidebars/admin/menus/spam_logs_menu.rb
@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module Admin
+ module Menus
+ class SpamLogsMenu < ::Sidebars::Menu
+ override :link
+ def link
+ admin_spam_logs_path
+ end
+
+ override :title
+ def title
+ s_('Admin|Spam Logs')
+ end
+
+ override :sprite_icon
+ def sprite_icon
+ 'spam'
+ end
+
+ override :render?
+ def render?
+ current_user && current_user.can_admin_all_resources? && anti_spam_service_enabled?
+ end
+
+ override :active_routes
+ def active_routes
+ { controller: :spam_logs }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/sidebars/admin/menus/system_hooks_menu.rb b/lib/sidebars/admin/menus/system_hooks_menu.rb
new file mode 100644
index 00000000000..494b0392400
--- /dev/null
+++ b/lib/sidebars/admin/menus/system_hooks_menu.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module Admin
+ module Menus
+ class SystemHooksMenu < ::Sidebars::Admin::BaseMenu
+ override :link
+ def link
+ admin_hooks_path
+ end
+
+ override :title
+ def title
+ s_('Admin|System Hooks')
+ end
+
+ override :sprite_icon
+ def sprite_icon
+ 'hook'
+ end
+
+ override :active_routes
+ def active_routes
+ { controller: :hooks }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/sidebars/admin/panel.rb b/lib/sidebars/admin/panel.rb
new file mode 100644
index 00000000000..4f14c4ffb49
--- /dev/null
+++ b/lib/sidebars/admin/panel.rb
@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module Admin
+ class Panel < ::Sidebars::Panel
+ override :configure_menus
+ def configure_menus
+ super
+ add_menus
+ end
+
+ override :render_raw_scope_menu_partial
+ def render_raw_scope_menu_partial
+ "shared/nav/admin_scope_header"
+ end
+
+ override :aria_label
+ def aria_label
+ s_("Admin|Admin Area")
+ end
+
+ override :super_sidebar_context_header
+ def super_sidebar_context_header
+ @super_sidebar_context_header ||= {
+ title: aria_label,
+ icon: 'admin'
+ }
+ end
+
+ def add_menus
+ add_menu(Sidebars::Admin::Menus::AdminOverviewMenu.new(context))
+ add_menu(Sidebars::Admin::Menus::CiCdMenu.new(context))
+ add_menu(Sidebars::Admin::Menus::AnalyticsMenu.new(context))
+ add_menu(Sidebars::Admin::Menus::MonitoringMenu.new(context))
+ add_menu(Sidebars::Admin::Menus::MessagesMenu.new(context))
+ add_menu(Sidebars::Admin::Menus::SystemHooksMenu.new(context))
+ add_menu(Sidebars::Admin::Menus::ApplicationsMenu.new(context))
+ add_menu(Sidebars::Admin::Menus::AbuseReportsMenu.new(context))
+ add_menu(Sidebars::Admin::Menus::KubernetesMenu.new(context))
+ add_menu(Sidebars::Admin::Menus::SpamLogsMenu.new(context))
+ add_menu(Sidebars::Admin::Menus::DeployKeysMenu.new(context))
+ add_menu(Sidebars::Admin::Menus::LabelsMenu.new(context))
+ add_menu(Sidebars::Admin::Menus::AdminSettingsMenu.new(context))
+ end
+ end
+ end
+end
+
+Sidebars::Admin::Panel.prepend_mod_with('Sidebars::Admin::Panel')
diff --git a/lib/sidebars/concerns/render_if_logged_in.rb b/lib/sidebars/concerns/render_if_logged_in.rb
new file mode 100644
index 00000000000..221af7df23b
--- /dev/null
+++ b/lib/sidebars/concerns/render_if_logged_in.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module Concerns
+ module RenderIfLoggedIn
+ def render?
+ !!context.current_user
+ end
+ end
+ end
+end
diff --git a/lib/sidebars/concerns/super_sidebar_panel.rb b/lib/sidebars/concerns/super_sidebar_panel.rb
new file mode 100644
index 00000000000..0afe01f926d
--- /dev/null
+++ b/lib/sidebars/concerns/super_sidebar_panel.rb
@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module Concerns
+ # Contains helper methods aid conversion of a "normal" panel
+ # into a Super Sidebar Panel
+ module SuperSidebarPanel
+ # Picks menus from a list and adds them to the current menu list
+ # if they should be picked into the super sidebar
+ def pick_from_old_menus(old_menus)
+ old_menus.select! do |menu|
+ next true unless menu.pick_into_super_sidebar?
+
+ add_menu(menu)
+ false
+ end
+ end
+
+ def transform_old_menus(current_menus, *old_menus)
+ old_menus.each do |menu|
+ next unless menu.render?
+
+ menu.renderable_items.each { |item| add_menu_item_to_super_sidebar_parent(current_menus, item) }
+
+ menu_item_args = menu.serialize_as_menu_item_args
+
+ next if menu_item_args.nil?
+
+ add_menu_item_to_super_sidebar_parent(
+ current_menus, ::Sidebars::MenuItem.new(**menu_item_args)
+ )
+ end
+ end
+
+ private
+
+ # Finds a menu_items super sidebar parent and adds the item to that menu
+ # Handles:
+ # - parent == nil, or parent not being part of the panel:
+ # we assume that the menu item hasn't been categorized yet
+ # - parent == ::Sidebars::NilMenuItem, the item explicitly is supposed to be removed
+ def add_menu_item_to_super_sidebar_parent(menus, menu_item)
+ parent = menu_item.super_sidebar_parent || ::Sidebars::UncategorizedMenu
+ return if parent == ::Sidebars::NilMenuItem
+
+ idx = index_of(menus, parent) || index_of(menus, ::Sidebars::UncategorizedMenu)
+ return unless idx
+
+ menus[idx].replace_placeholder(menu_item)
+ end
+ end
+ end
+end
diff --git a/lib/sidebars/context.rb b/lib/sidebars/context.rb
index d9ac2705aaf..b49776bed10 100644
--- a/lib/sidebars/context.rb
+++ b/lib/sidebars/context.rb
@@ -6,16 +6,19 @@
# values where the logic is in helpers.
module Sidebars
class Context
- attr_reader :current_user, :container
+ attr_reader :current_user, :container, :route_is_active, :is_super_sidebar
def initialize(current_user:, container:, **args)
@current_user = current_user
@container = container
+ @is_super_sidebar = false
args.each do |key, value|
singleton_class.public_send(:attr_reader, key) # rubocop:disable GitlabSecurity/PublicSend
instance_variable_set("@#{key}", value)
end
+
+ @route_is_active ||= ->(_) { false }
end
end
end
diff --git a/lib/sidebars/explore/menus/groups_menu.rb b/lib/sidebars/explore/menus/groups_menu.rb
new file mode 100644
index 00000000000..542da0ad7fd
--- /dev/null
+++ b/lib/sidebars/explore/menus/groups_menu.rb
@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module Explore
+ module Menus
+ class GroupsMenu < ::Sidebars::Menu
+ override :link
+ def link
+ explore_groups_path
+ end
+
+ override :title
+ def title
+ _('Groups')
+ end
+
+ override :sprite_icon
+ def sprite_icon
+ 'group'
+ end
+
+ override :render?
+ def render?
+ true
+ end
+
+ override :active_routes
+ def active_routes
+ { controller: ['explore/groups'] }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/sidebars/explore/menus/projects_menu.rb b/lib/sidebars/explore/menus/projects_menu.rb
new file mode 100644
index 00000000000..c34bd7ed3da
--- /dev/null
+++ b/lib/sidebars/explore/menus/projects_menu.rb
@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module Explore
+ module Menus
+ class ProjectsMenu < ::Sidebars::Menu
+ override :link
+ def link
+ explore_projects_path
+ end
+
+ override :title
+ def title
+ _('Projects')
+ end
+
+ override :sprite_icon
+ def sprite_icon
+ 'project'
+ end
+
+ override :render?
+ def render?
+ true
+ end
+
+ override :active_routes
+ def active_routes
+ { page: [link, explore_root_path, starred_explore_projects_path, trending_explore_projects_path] }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/sidebars/explore/menus/snippets_menu.rb b/lib/sidebars/explore/menus/snippets_menu.rb
new file mode 100644
index 00000000000..67b852258a7
--- /dev/null
+++ b/lib/sidebars/explore/menus/snippets_menu.rb
@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module Explore
+ module Menus
+ class SnippetsMenu < ::Sidebars::Menu
+ override :link
+ def link
+ explore_snippets_path
+ end
+
+ override :title
+ def title
+ _('Snippets')
+ end
+
+ override :sprite_icon
+ def sprite_icon
+ 'snippet'
+ end
+
+ override :render?
+ def render?
+ true
+ end
+
+ override :active_routes
+ def active_routes
+ { controller: ['explore/snippets'] }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/sidebars/explore/menus/topics_menu.rb b/lib/sidebars/explore/menus/topics_menu.rb
new file mode 100644
index 00000000000..5e0a7897873
--- /dev/null
+++ b/lib/sidebars/explore/menus/topics_menu.rb
@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module Explore
+ module Menus
+ class TopicsMenu < ::Sidebars::Menu
+ override :link
+ def link
+ topics_explore_projects_path
+ end
+
+ override :title
+ def title
+ _('Topics')
+ end
+
+ override :sprite_icon
+ def sprite_icon
+ 'labels'
+ end
+
+ override :render?
+ def render?
+ true
+ end
+
+ override :active_routes
+ def active_routes
+ { page: link, path: 'projects#topic' }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/sidebars/explore/panel.rb b/lib/sidebars/explore/panel.rb
new file mode 100644
index 00000000000..9a585a99705
--- /dev/null
+++ b/lib/sidebars/explore/panel.rb
@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module Explore
+ class Panel < ::Sidebars::Panel
+ override :configure_menus
+ def configure_menus
+ add_menus
+ end
+
+ override :aria_label
+ def aria_label
+ _('Explore')
+ end
+
+ override :render_raw_scope_menu_partial
+ def render_raw_scope_menu_partial
+ "shared/nav/explore_scope_header"
+ end
+
+ override :super_sidebar_context_header
+ def super_sidebar_context_header
+ {
+ title: aria_label,
+ icon: 'compass'
+ }
+ end
+
+ private
+
+ def add_menus
+ add_menu(Sidebars::Explore::Menus::ProjectsMenu.new(context))
+ add_menu(Sidebars::Explore::Menus::GroupsMenu.new(context))
+ add_menu(Sidebars::Explore::Menus::TopicsMenu.new(context))
+ add_menu(Sidebars::Explore::Menus::SnippetsMenu.new(context))
+ end
+ end
+ end
+end
diff --git a/lib/sidebars/groups/menus/ci_cd_menu.rb b/lib/sidebars/groups/menus/ci_cd_menu.rb
index 0c2995f95e6..8fa90a3f923 100644
--- a/lib/sidebars/groups/menus/ci_cd_menu.rb
+++ b/lib/sidebars/groups/menus/ci_cd_menu.rb
@@ -21,6 +21,11 @@ module Sidebars
'rocket'
end
+ override :serialize_as_menu_item_args
+ def serialize_as_menu_item_args
+ nil
+ end
+
private
def runners_menu_item
@@ -29,6 +34,7 @@ module Sidebars
::Sidebars::MenuItem.new(
title: _('Runners'),
link: group_runners_path(context.group),
+ super_sidebar_parent: ::Sidebars::Groups::SuperSidebarMenus::BuildMenu,
active_routes: { controller: 'groups/runners' },
item_id: :runners
)
diff --git a/lib/sidebars/groups/menus/customer_relations_menu.rb b/lib/sidebars/groups/menus/customer_relations_menu.rb
index e0772cfe403..ac25cb312cd 100644
--- a/lib/sidebars/groups/menus/customer_relations_menu.rb
+++ b/lib/sidebars/groups/menus/customer_relations_menu.rb
@@ -29,12 +29,18 @@ module Sidebars
can_read_contact? || can_read_organization?
end
+ override :serialize_as_menu_item_args
+ def serialize_as_menu_item_args
+ nil
+ end
+
private
def contacts_menu_item
::Sidebars::MenuItem.new(
- title: _('Contacts'),
+ title: context.is_super_sidebar ? _('Customer contacts') : _('Contacts'),
link: group_crm_contacts_path(context.group),
+ super_sidebar_parent: ::Sidebars::Groups::SuperSidebarMenus::PlanMenu,
active_routes: { controller: 'groups/crm/contacts' },
item_id: :crm_contacts
)
@@ -42,8 +48,9 @@ module Sidebars
def organizations_menu_item
::Sidebars::MenuItem.new(
- title: _('Organizations'),
+ title: context.is_super_sidebar ? _('Customer organizations') : _('Organizations'),
link: group_crm_organizations_path(context.group),
+ super_sidebar_parent: ::Sidebars::Groups::SuperSidebarMenus::PlanMenu,
active_routes: { controller: 'groups/crm/organizations' },
item_id: :crm_organizations
)
diff --git a/lib/sidebars/groups/menus/group_information_menu.rb b/lib/sidebars/groups/menus/group_information_menu.rb
index 3ce99e14a04..64295620def 100644
--- a/lib/sidebars/groups/menus/group_information_menu.rb
+++ b/lib/sidebars/groups/menus/group_information_menu.rb
@@ -28,6 +28,11 @@ module Sidebars
{ path: 'groups#subgroups' }
end
+ override :serialize_as_menu_item_args
+ def serialize_as_menu_item_args
+ nil
+ end
+
private
def activity_menu_item
@@ -38,6 +43,7 @@ module Sidebars
::Sidebars::MenuItem.new(
title: _('Activity'),
link: activity_group_path(context.group),
+ super_sidebar_parent: ::Sidebars::Groups::SuperSidebarMenus::ManageMenu,
active_routes: { path: 'groups#activity' },
item_id: :activity
)
@@ -51,6 +57,7 @@ module Sidebars
::Sidebars::MenuItem.new(
title: _('Labels'),
link: group_labels_path(context.group),
+ super_sidebar_parent: ::Sidebars::Groups::SuperSidebarMenus::ManageMenu,
active_routes: { controller: :labels },
item_id: :labels
)
@@ -64,6 +71,8 @@ module Sidebars
::Sidebars::MenuItem.new(
title: _('Members'),
link: group_group_members_path(context.group),
+ sprite_icon: nil,
+ super_sidebar_parent: ::Sidebars::Groups::SuperSidebarMenus::ManageMenu,
active_routes: { path: 'group_members#index' },
item_id: :members
)
diff --git a/lib/sidebars/groups/menus/invite_team_members_menu.rb b/lib/sidebars/groups/menus/invite_team_members_menu.rb
deleted file mode 100644
index 0779b696061..00000000000
--- a/lib/sidebars/groups/menus/invite_team_members_menu.rb
+++ /dev/null
@@ -1,46 +0,0 @@
-# frozen_string_literal: true
-
-module Sidebars
- module Groups
- module Menus
- class InviteTeamMembersMenu < ::Sidebars::Menu
- override :title
- def title
- s_('InviteMember|Invite members')
- end
-
- override :render?
- def render?
- can?(context.current_user, :admin_group_member, context.group) && all_valid_members.size <= 1
- end
-
- override :menu_partial
- def menu_partial
- 'groups/invite_members_side_nav_link'
- end
-
- override :menu_partial_options
- def menu_partial_options
- {
- group: context.group,
- title: title,
- sidebar_menu: self
- }
- end
-
- override :extra_nav_link_html_options
- def extra_nav_link_html_options
- {
- 'data-test-id': 'side-nav-invite-members'
- }
- end
-
- private
-
- def all_valid_members
- GroupMembersFinder.new(context.group, context.current_user).execute
- end
- end
- end
- end
-end
diff --git a/lib/sidebars/groups/menus/issues_menu.rb b/lib/sidebars/groups/menus/issues_menu.rb
index 4044cb1c716..ddc930d56ff 100644
--- a/lib/sidebars/groups/menus/issues_menu.rb
+++ b/lib/sidebars/groups/menus/issues_menu.rb
@@ -49,12 +49,24 @@ module Sidebars
}
end
+ override :serialize_as_menu_item_args
+ def serialize_as_menu_item_args
+ super.merge({
+ active_routes: list_menu_item.active_routes,
+ pill_count: pill_count,
+ has_pill: has_pill?,
+ super_sidebar_parent: ::Sidebars::Groups::SuperSidebarMenus::PlanMenu,
+ item_id: :group_issue_list
+ })
+ end
+
private
def list_menu_item
::Sidebars::MenuItem.new(
title: _('List'),
link: issues_group_path(context.group),
+ super_sidebar_parent: ::Sidebars::NilMenuItem,
active_routes: { path: 'groups#issues' },
container_html_options: { aria: { label: _('Issues') } },
item_id: :issue_list
@@ -66,13 +78,18 @@ module Sidebars
return ::Sidebars::NilMenuItem.new(item_id: :boards)
end
- title = context.group.multiple_issue_boards_available? ? s_('IssueBoards|Boards') : s_('IssueBoards|Board')
+ title = if context.is_super_sidebar
+ context.group.multiple_issue_boards_available? ? s_('Issue boards') : s_('Issue board')
+ else
+ context.group.multiple_issue_boards_available? ? s_('IssueBoards|Boards') : s_('IssueBoards|Board')
+ end
::Sidebars::MenuItem.new(
title: title,
link: group_boards_path(context.group),
+ super_sidebar_parent: ::Sidebars::Groups::SuperSidebarMenus::PlanMenu,
active_routes: { path: %w[boards#index boards#show] },
- item_id: :boards
+ item_id: context.is_super_sidebar ? :issue_boards : :boards
)
end
@@ -84,6 +101,7 @@ module Sidebars
::Sidebars::MenuItem.new(
title: _('Milestones'),
link: group_milestones_path(context.group),
+ super_sidebar_parent: ::Sidebars::Groups::SuperSidebarMenus::PlanMenu,
active_routes: { path: 'milestones#index' },
item_id: :milestones
)
diff --git a/lib/sidebars/groups/menus/kubernetes_menu.rb b/lib/sidebars/groups/menus/kubernetes_menu.rb
index 0d845978a93..a7c14148230 100644
--- a/lib/sidebars/groups/menus/kubernetes_menu.rb
+++ b/lib/sidebars/groups/menus/kubernetes_menu.rb
@@ -38,6 +38,14 @@ module Sidebars
def active_routes
{ controller: :clusters }
end
+
+ override :serialize_as_menu_item_args
+ def serialize_as_menu_item_args
+ super.merge({
+ super_sidebar_parent: ::Sidebars::Groups::SuperSidebarMenus::OperationsMenu,
+ item_id: :group_kubernetes_clusters
+ })
+ end
end
end
end
diff --git a/lib/sidebars/groups/menus/merge_requests_menu.rb b/lib/sidebars/groups/menus/merge_requests_menu.rb
index 050cba07641..356c823add9 100644
--- a/lib/sidebars/groups/menus/merge_requests_menu.rb
+++ b/lib/sidebars/groups/menus/merge_requests_menu.rb
@@ -52,6 +52,16 @@ module Sidebars
def active_routes
{ path: 'groups#merge_requests' }
end
+
+ override :serialize_as_menu_item_args
+ def serialize_as_menu_item_args
+ super.merge({
+ pill_count: pill_count,
+ has_pill: has_pill?,
+ super_sidebar_parent: ::Sidebars::Groups::SuperSidebarMenus::CodeMenu,
+ item_id: :group_merge_request_list
+ })
+ end
end
end
end
diff --git a/lib/sidebars/groups/menus/observability_menu.rb b/lib/sidebars/groups/menus/observability_menu.rb
index d85efb1a002..268528356f1 100644
--- a/lib/sidebars/groups/menus/observability_menu.rb
+++ b/lib/sidebars/groups/menus/observability_menu.rb
@@ -6,8 +6,11 @@ module Sidebars
class ObservabilityMenu < ::Sidebars::Menu
override :configure_menu_items
def configure_menu_items
- add_item(explore_menu_item)
- add_item(datasources_menu_item)
+ add_item(explore_menu_item) if Gitlab::Observability.allowed_for_action?(context.current_user, context.group,
+ :explore)
+
+ add_item(datasources_menu_item) if Gitlab::Observability.allowed_for_action?(context.current_user,
+ context.group, :datasources)
end
override :title
@@ -22,7 +25,12 @@ module Sidebars
override :render?
def render?
- Gitlab::Observability.observability_enabled?(context.current_user, context.group)
+ Gitlab::Observability.allowed_for_action?(context.current_user, context.group, :explore)
+ end
+
+ override :serialize_as_menu_item_args
+ def serialize_as_menu_item_args
+ nil
end
private
@@ -31,6 +39,7 @@ module Sidebars
::Sidebars::MenuItem.new(
title: s_('Observability|Dashboards'),
link: group_observability_dashboards_path(context.group),
+ super_sidebar_parent: ::Sidebars::Groups::SuperSidebarMenus::MonitorMenu,
active_routes: { path: 'groups/observability#dashboards' },
item_id: :dashboards
)
@@ -40,6 +49,7 @@ module Sidebars
::Sidebars::MenuItem.new(
title: s_('Observability|Explore telemetry data'),
link: group_observability_explore_path(context.group),
+ super_sidebar_parent: ::Sidebars::Groups::SuperSidebarMenus::MonitorMenu,
active_routes: { path: 'groups/observability#explore' },
item_id: :explore
)
@@ -49,6 +59,7 @@ module Sidebars
::Sidebars::MenuItem.new(
title: s_('Observability|Data sources'),
link: group_observability_datasources_path(context.group),
+ super_sidebar_parent: ::Sidebars::Groups::SuperSidebarMenus::MonitorMenu,
active_routes: { path: 'groups/observability#datasources' },
item_id: :datasources
)
@@ -58,6 +69,7 @@ module Sidebars
::Sidebars::MenuItem.new(
title: s_('Observability|Manage dashboards'),
link: group_observability_manage_path(context.group),
+ super_sidebar_parent: ::Sidebars::Groups::SuperSidebarMenus::MonitorMenu,
active_routes: { path: 'groups/observability#manage' },
item_id: :manage
)
diff --git a/lib/sidebars/groups/menus/packages_registries_menu.rb b/lib/sidebars/groups/menus/packages_registries_menu.rb
index e115ca669d4..68c8e9675a7 100644
--- a/lib/sidebars/groups/menus/packages_registries_menu.rb
+++ b/lib/sidebars/groups/menus/packages_registries_menu.rb
@@ -23,6 +23,11 @@ module Sidebars
'package'
end
+ override :serialize_as_menu_item_args
+ def serialize_as_menu_item_args
+ nil
+ end
+
private
def packages_registry_menu_item
@@ -31,6 +36,7 @@ module Sidebars
::Sidebars::MenuItem.new(
title: _('Package Registry'),
link: group_packages_path(context.group),
+ super_sidebar_parent: ::Sidebars::Groups::SuperSidebarMenus::OperationsMenu,
active_routes: { controller: 'groups/packages' },
item_id: :packages_registry
)
@@ -44,6 +50,7 @@ module Sidebars
::Sidebars::MenuItem.new(
title: _('Container Registry'),
link: group_container_registries_path(context.group),
+ super_sidebar_parent: ::Sidebars::Groups::SuperSidebarMenus::OperationsMenu,
active_routes: { controller: 'groups/registry/repositories' },
item_id: :container_registry
)
@@ -59,6 +66,7 @@ module Sidebars
::Sidebars::MenuItem.new(
title: _('Harbor Registry'),
link: group_harbor_repositories_path(context.group),
+ super_sidebar_parent: ::Sidebars::Groups::SuperSidebarMenus::OperationsMenu,
active_routes: { controller: 'groups/harbor/repositories' },
item_id: :harbor_registry
)
@@ -74,6 +82,7 @@ module Sidebars
::Sidebars::MenuItem.new(
title: _('Dependency Proxy'),
link: group_dependency_proxy_path(context.group),
+ super_sidebar_parent: ::Sidebars::Groups::SuperSidebarMenus::OperationsMenu,
active_routes: { controller: 'groups/dependency_proxies' },
item_id: :dependency_proxy
)
diff --git a/lib/sidebars/groups/menus/scope_menu.rb b/lib/sidebars/groups/menus/scope_menu.rb
index 6ce43491343..5505f56a6d3 100644
--- a/lib/sidebars/groups/menus/scope_menu.rb
+++ b/lib/sidebars/groups/menus/scope_menu.rb
@@ -16,7 +16,7 @@ module Sidebars
override :active_routes
def active_routes
- { path: %w[groups#show groups#details] }
+ { path: %w[groups#show groups#details groups#new projects#new] }
end
override :extra_nav_link_html_options
@@ -32,6 +32,16 @@ module Sidebars
def render?
true
end
+
+ override :serialize_as_menu_item_args
+ def serialize_as_menu_item_args
+ super.merge({
+ title: _('Group overview'),
+ sprite_icon: 'group',
+ super_sidebar_parent: ::Sidebars::StaticMenu,
+ item_id: :group_overview
+ })
+ end
end
end
end
diff --git a/lib/sidebars/groups/menus/settings_menu.rb b/lib/sidebars/groups/menus/settings_menu.rb
index 5b81f22c796..b8f345c1ed5 100644
--- a/lib/sidebars/groups/menus/settings_menu.rb
+++ b/lib/sidebars/groups/menus/settings_menu.rb
@@ -47,6 +47,16 @@ module Sidebars
}
end
+ override :pick_into_super_sidebar?
+ def pick_into_super_sidebar?
+ true
+ end
+
+ override :separated?
+ def separated?
+ true
+ end
+
private
def general_menu_item
diff --git a/lib/sidebars/groups/panel.rb b/lib/sidebars/groups/panel.rb
index e8b815bdce7..77ca51ddf92 100644
--- a/lib/sidebars/groups/panel.rb
+++ b/lib/sidebars/groups/panel.rb
@@ -16,22 +16,12 @@ module Sidebars
add_menu(Sidebars::Groups::Menus::PackagesRegistriesMenu.new(context))
add_menu(Sidebars::Groups::Menus::CustomerRelationsMenu.new(context))
add_menu(Sidebars::Groups::Menus::SettingsMenu.new(context))
- add_invite_members_menu
end
override :aria_label
def aria_label
context.group.subgroup? ? _('Subgroup navigation') : _('Group navigation')
end
-
- private
-
- def add_invite_members_menu
- experiment(:invite_members_in_side_nav, group: context.group) do |e|
- e.control {}
- e.candidate { add_menu(Sidebars::Groups::Menus::InviteTeamMembersMenu.new(context)) }
- end
- end
end
end
end
diff --git a/lib/sidebars/groups/super_sidebar_menus/analyze_menu.rb b/lib/sidebars/groups/super_sidebar_menus/analyze_menu.rb
new file mode 100644
index 00000000000..65393336797
--- /dev/null
+++ b/lib/sidebars/groups/super_sidebar_menus/analyze_menu.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module Groups
+ module SuperSidebarMenus
+ class AnalyzeMenu < ::Sidebars::Menu
+ override :title
+ def title
+ s_('Navigation|Analyze')
+ end
+
+ override :sprite_icon
+ def sprite_icon
+ 'chart'
+ end
+
+ override :configure_menu_items
+ def configure_menu_items
+ [
+ :cycle_analytics,
+ :ci_cd_analytics,
+ :contribution_analytics,
+ :devops_adoption,
+ :insights,
+ :issues_analytics,
+ :productivity_analytics,
+ :repository_analytics
+ ].each { |id| add_item(::Sidebars::NilMenuItem.new(item_id: id)) }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/sidebars/groups/super_sidebar_menus/build_menu.rb b/lib/sidebars/groups/super_sidebar_menus/build_menu.rb
new file mode 100644
index 00000000000..988f3d498b9
--- /dev/null
+++ b/lib/sidebars/groups/super_sidebar_menus/build_menu.rb
@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module Groups
+ module SuperSidebarMenus
+ class BuildMenu < ::Sidebars::Menu
+ override :title
+ def title
+ s_('Navigation|Build')
+ end
+
+ override :sprite_icon
+ def sprite_icon
+ 'rocket'
+ end
+
+ override :configure_menu_items
+ def configure_menu_items
+ [
+ :runners
+ ].each { |id| add_item(::Sidebars::NilMenuItem.new(item_id: id)) }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/sidebars/groups/super_sidebar_menus/code_menu.rb b/lib/sidebars/groups/super_sidebar_menus/code_menu.rb
new file mode 100644
index 00000000000..e102066b18d
--- /dev/null
+++ b/lib/sidebars/groups/super_sidebar_menus/code_menu.rb
@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module Groups
+ module SuperSidebarMenus
+ class CodeMenu < ::Sidebars::Menu
+ override :title
+ def title
+ s_('Navigation|Code')
+ end
+
+ override :sprite_icon
+ def sprite_icon
+ 'code'
+ end
+
+ override :configure_menu_items
+ def configure_menu_items
+ [
+ :group_merge_request_list
+ ].each { |id| add_item(::Sidebars::NilMenuItem.new(item_id: id)) }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/sidebars/groups/super_sidebar_menus/manage_menu.rb b/lib/sidebars/groups/super_sidebar_menus/manage_menu.rb
new file mode 100644
index 00000000000..f926892083b
--- /dev/null
+++ b/lib/sidebars/groups/super_sidebar_menus/manage_menu.rb
@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module Groups
+ module SuperSidebarMenus
+ class ManageMenu < ::Sidebars::Menu
+ override :title
+ def title
+ s_('Navigation|Manage')
+ end
+
+ override :sprite_icon
+ def sprite_icon
+ 'users'
+ end
+
+ override :configure_menu_items
+ def configure_menu_items
+ [
+ :activity,
+ :members,
+ :labels
+ ].each { |id| add_item(::Sidebars::NilMenuItem.new(item_id: id)) }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/sidebars/groups/super_sidebar_menus/monitor_menu.rb b/lib/sidebars/groups/super_sidebar_menus/monitor_menu.rb
new file mode 100644
index 00000000000..8ee0aaaa808
--- /dev/null
+++ b/lib/sidebars/groups/super_sidebar_menus/monitor_menu.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module Groups
+ module SuperSidebarMenus
+ class MonitorMenu < ::Sidebars::Menu
+ override :title
+ def title
+ s_('Navigation|Monitor')
+ end
+
+ override :sprite_icon
+ def sprite_icon
+ 'monitor'
+ end
+
+ override :configure_menu_items
+ def configure_menu_items
+ [
+ :explore,
+ :datasources
+ ].each { |id| add_item(::Sidebars::NilMenuItem.new(item_id: id)) }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/sidebars/groups/super_sidebar_menus/operations_menu.rb b/lib/sidebars/groups/super_sidebar_menus/operations_menu.rb
new file mode 100644
index 00000000000..fe17ada69e4
--- /dev/null
+++ b/lib/sidebars/groups/super_sidebar_menus/operations_menu.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module Groups
+ module SuperSidebarMenus
+ class OperationsMenu < ::Sidebars::Menu
+ override :title
+ def title
+ s_('Navigation|Operate')
+ end
+
+ override :sprite_icon
+ def sprite_icon
+ 'deployments'
+ end
+
+ override :configure_menu_items
+ def configure_menu_items
+ [
+ :dependency_proxy,
+ :packages_registry,
+ :container_registry,
+ :group_kubernetes_clusters
+ ].each { |id| add_item(::Sidebars::NilMenuItem.new(item_id: id)) }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/sidebars/groups/super_sidebar_menus/plan_menu.rb b/lib/sidebars/groups/super_sidebar_menus/plan_menu.rb
new file mode 100644
index 00000000000..bf122f930a2
--- /dev/null
+++ b/lib/sidebars/groups/super_sidebar_menus/plan_menu.rb
@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module Groups
+ module SuperSidebarMenus
+ class PlanMenu < ::Sidebars::Menu
+ override :title
+ def title
+ s_('Navigation|Plan')
+ end
+
+ override :sprite_icon
+ def sprite_icon
+ 'planning'
+ end
+
+ override :configure_menu_items
+ def configure_menu_items
+ [
+ :group_issue_list,
+ :group_epic_list,
+ :issue_boards,
+ :epic_boards,
+ :roadmap,
+ :milestones,
+ :iterations,
+ :group_wiki,
+ :crm_contacts,
+ :crm_organizations
+ ].each { |id| add_item(::Sidebars::NilMenuItem.new(item_id: id)) }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/sidebars/groups/super_sidebar_menus/secure_menu.rb b/lib/sidebars/groups/super_sidebar_menus/secure_menu.rb
new file mode 100644
index 00000000000..c79e7e379d0
--- /dev/null
+++ b/lib/sidebars/groups/super_sidebar_menus/secure_menu.rb
@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module Groups
+ module SuperSidebarMenus
+ class SecureMenu < ::Sidebars::Menu
+ override :title
+ def title
+ s_('Navigation|Secure')
+ end
+
+ override :sprite_icon
+ def sprite_icon
+ 'shield'
+ end
+
+ override :configure_menu_items
+ def configure_menu_items
+ [
+ :security_dashboard,
+ :vulnerability_report,
+ :audit_events,
+ :compliance,
+ :scan_policies
+ ].each { |id| add_item(::Sidebars::NilMenuItem.new(item_id: id)) }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/sidebars/groups/super_sidebar_panel.rb b/lib/sidebars/groups/super_sidebar_panel.rb
new file mode 100644
index 00000000000..03af904d99d
--- /dev/null
+++ b/lib/sidebars/groups/super_sidebar_panel.rb
@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module Groups
+ class SuperSidebarPanel < ::Sidebars::Groups::Panel
+ include ::Sidebars::Concerns::SuperSidebarPanel
+ extend ::Gitlab::Utils::Override
+
+ override :configure_menus
+ def configure_menus
+ super
+ old_menus = @menus
+ @menus = []
+
+ add_menu(Sidebars::StaticMenu.new(context))
+ add_menu(Sidebars::Groups::SuperSidebarMenus::ManageMenu.new(context))
+ add_menu(Sidebars::Groups::SuperSidebarMenus::PlanMenu.new(context))
+ add_menu(Sidebars::Groups::SuperSidebarMenus::CodeMenu.new(context))
+ add_menu(Sidebars::Groups::SuperSidebarMenus::BuildMenu.new(context))
+ add_menu(Sidebars::Groups::SuperSidebarMenus::SecureMenu.new(context))
+ add_menu(Sidebars::Groups::SuperSidebarMenus::OperationsMenu.new(context))
+ add_menu(Sidebars::Groups::SuperSidebarMenus::MonitorMenu.new(context))
+ add_menu(Sidebars::Groups::SuperSidebarMenus::AnalyzeMenu.new(context))
+
+ pick_from_old_menus(old_menus)
+
+ insert_menu_before(
+ Sidebars::Groups::Menus::SettingsMenu,
+ Sidebars::UncategorizedMenu.new(context)
+ )
+
+ transform_old_menus(@menus, @scope_menu, *old_menus)
+ end
+
+ override :super_sidebar_context_header
+ def super_sidebar_context_header
+ {
+ title: context.group.name,
+ avatar: context.group.avatar_url,
+ id: context.group.id
+ }
+ end
+ end
+ end
+end
diff --git a/lib/sidebars/menu.rb b/lib/sidebars/menu.rb
index dfd88c99a0c..5f9255c06d0 100644
--- a/lib/sidebars/menu.rb
+++ b/lib/sidebars/menu.rb
@@ -66,6 +66,46 @@ module Sidebars
@renderable_items ||= @items.select(&:render?)
end
+ # Defines whether menu is separated from others with a top separator
+ def separated?
+ false
+ end
+
+ # Returns a tree-like representation of itself and all
+ # renderable menu entries, with additional information
+ # on whether the item(s) have an active route
+ def serialize_for_super_sidebar
+ items = serialize_items_for_super_sidebar
+ is_active = @context.route_is_active.call(active_routes) || items.any? { |item| item[:is_active] }
+
+ {
+ title: title,
+ icon: sprite_icon,
+ link: link,
+ is_active: is_active,
+ pill_count: has_pill? ? pill_count : nil,
+ items: items,
+ separated: separated?
+ }
+ end
+
+ # Returns an array of renderable menu entries,
+ # with additional information on whether the item
+ # has an active route
+ def serialize_items_for_super_sidebar
+ # All renderable menu entries
+ renderable_items.map do |entry|
+ entry.serialize_for_super_sidebar.tap do |item|
+ active_routes = item.delete(:active_routes)
+ item[:is_active] = active_routes ? @context.route_is_active.call(active_routes) : false
+ end
+ end
+ end
+
+ def pick_into_super_sidebar?
+ false
+ end
+
# Returns whether the menu has any renderable menu item
def has_renderable_items?
renderable_items.any?
@@ -83,6 +123,15 @@ module Sidebars
insert_element_after(@items, after_item, new_item)
end
+ def replace_placeholder(item)
+ idx = @items.index { |e| e.item_id == item.item_id && e.is_a?(::Sidebars::NilMenuItem) }
+ if idx.nil?
+ add_item(item)
+ else
+ replace_element(@items, item.item_id, item)
+ end
+ end
+
override :container_html_options
def container_html_options
super.tap do |html_options|
@@ -93,6 +142,17 @@ module Sidebars
end
end
+ # Sometimes we want to convert a top-level Menu (e.g. Wiki/Snippets)
+ # to a MenuItem. This serializer is used in order to enable that conversion
+ def serialize_as_menu_item_args
+ {
+ title: title,
+ link: link,
+ active_routes: active_routes,
+ container_html_options: container_html_options
+ }
+ end
+
private
override :index_of
diff --git a/lib/sidebars/menu_item.rb b/lib/sidebars/menu_item.rb
index efdedf6c3bd..227a6970b2c 100644
--- a/lib/sidebars/menu_item.rb
+++ b/lib/sidebars/menu_item.rb
@@ -4,11 +4,11 @@ module Sidebars
class MenuItem
include ::Sidebars::Concerns::LinkWithHtmlOptions
- attr_reader :title, :link, :active_routes, :item_id, :container_html_options, :sprite_icon, :sprite_icon_html_options, :hint_html_options, :has_pill, :pill_count
+ attr_reader :title, :link, :active_routes, :item_id, :container_html_options, :sprite_icon, :sprite_icon_html_options, :hint_html_options, :has_pill, :pill_count, :super_sidebar_parent
alias_method :has_pill?, :has_pill
# rubocop: disable Metrics/ParameterLists
- def initialize(title:, link:, active_routes:, item_id: nil, container_html_options: {}, sprite_icon: nil, sprite_icon_html_options: {}, hint_html_options: {}, has_pill: false, pill_count: nil)
+ def initialize(title:, link:, active_routes:, item_id: nil, container_html_options: {}, sprite_icon: nil, sprite_icon_html_options: {}, hint_html_options: {}, has_pill: false, pill_count: nil, super_sidebar_parent: nil)
@title = title
@link = link
@active_routes = active_routes
@@ -19,6 +19,7 @@ module Sidebars
@hint_html_options = hint_html_options
@has_pill = has_pill
@pill_count = pill_count
+ @super_sidebar_parent = super_sidebar_parent
end
# rubocop: enable Metrics/ParameterLists
@@ -30,6 +31,25 @@ module Sidebars
true
end
+ def serialize_for_super_sidebar
+ {
+ id: item_id,
+ title: title,
+ icon: sprite_icon,
+ link: link,
+ active_routes: active_routes,
+ pill_count: has_pill ? pill_count : nil,
+ link_classes: container_html_options[:class]
+ # Check whether support is needed for the following properties,
+ # in order to get feature parity with the HAML renderer
+ # https://gitlab.com/gitlab-org/gitlab/-/issues/391864
+ #
+ # container_html_options
+ # hint_html_options
+ # nav_link_html_options
+ }
+ end
+
def nav_link_html_options
{
data: {
diff --git a/lib/sidebars/panel.rb b/lib/sidebars/panel.rb
index 2a172cfffe3..4c0d4caf81e 100644
--- a/lib/sidebars/panel.rb
+++ b/lib/sidebars/panel.rb
@@ -4,7 +4,6 @@ module Sidebars
class Panel
extend ::Gitlab::Utils::Override
include ::Sidebars::Concerns::PositionableList
- include Gitlab::Experiment::Dsl
attr_reader :context, :scope_menu, :hidden_menu
@@ -61,6 +60,16 @@ module Sidebars
@renderable_menus ||= @menus.select(&:render?)
end
+ # Serializes every renderable menu and returns a flattened result
+ def super_sidebar_menu_items
+ @super_sidebar_menu_items ||= renderable_menus
+ .flat_map(&:serialize_for_super_sidebar)
+ end
+
+ def super_sidebar_context_header
+ raise NotImplementedError
+ end
+
def container
context.container
end
diff --git a/lib/sidebars/projects/menus/analytics_menu.rb b/lib/sidebars/projects/menus/analytics_menu.rb
index 643b7ebcd5a..96b50cdfcd1 100644
--- a/lib/sidebars/projects/menus/analytics_menu.rb
+++ b/lib/sidebars/projects/menus/analytics_menu.rb
@@ -41,6 +41,11 @@ module Sidebars
'chart'
end
+ override :serialize_as_menu_item_args
+ def serialize_as_menu_item_args
+ nil
+ end
+
private
def ci_cd_analytics_menu_item
@@ -52,8 +57,9 @@ module Sidebars
end
::Sidebars::MenuItem.new(
- title: _('CI/CD'),
+ title: context.is_super_sidebar ? _('CI/CD analytics') : _('CI/CD'),
link: charts_project_pipelines_path(context.project),
+ super_sidebar_parent: ::Sidebars::Projects::SuperSidebarMenus::AnalyzeMenu,
active_routes: { path: 'pipelines#charts' },
item_id: :ci_cd_analytics
)
@@ -65,8 +71,9 @@ module Sidebars
end
::Sidebars::MenuItem.new(
- title: _('Repository'),
+ title: context.is_super_sidebar ? _('Repository analytics') : _('Repository'),
link: charts_project_graph_path(context.project, context.current_ref),
+ super_sidebar_parent: ::Sidebars::Projects::SuperSidebarMenus::AnalyzeMenu,
container_html_options: { class: 'shortcuts-repository-charts' },
active_routes: { path: 'graphs#charts' },
item_id: :repository_analytics
@@ -80,8 +87,9 @@ module Sidebars
end
::Sidebars::MenuItem.new(
- title: _('Value stream'),
+ title: context.is_super_sidebar ? _('Value stream analytics') : _('Value stream'),
link: project_cycle_analytics_path(context.project),
+ super_sidebar_parent: ::Sidebars::Projects::SuperSidebarMenus::AnalyzeMenu,
container_html_options: { class: 'shortcuts-project-cycle-analytics' },
active_routes: { path: 'cycle_analytics#show' },
item_id: :cycle_analytics
diff --git a/lib/sidebars/projects/menus/ci_cd_menu.rb b/lib/sidebars/projects/menus/ci_cd_menu.rb
index 5df99bb9d84..02596b16cfa 100644
--- a/lib/sidebars/projects/menus/ci_cd_menu.rb
+++ b/lib/sidebars/projects/menus/ci_cd_menu.rb
@@ -39,12 +39,18 @@ module Sidebars
'rocket'
end
+ override :serialize_as_menu_item_args
+ def serialize_as_menu_item_args
+ nil
+ end
+
private
def pipelines_menu_item
::Sidebars::MenuItem.new(
title: _('Pipelines'),
link: project_pipelines_path(context.project),
+ super_sidebar_parent: ::Sidebars::Projects::SuperSidebarMenus::BuildMenu,
container_html_options: { class: 'shortcuts-pipelines' },
active_routes: { path: pipelines_routes },
item_id: :pipelines
@@ -73,8 +79,9 @@ module Sidebars
}
::Sidebars::MenuItem.new(
- title: s_('Pipelines|Editor'),
+ title: context.is_super_sidebar ? _('Pipeline editor') : s_('Pipelines|Editor'),
link: project_ci_pipeline_editor_path(context.project, params),
+ super_sidebar_parent: ::Sidebars::Projects::SuperSidebarMenus::BuildMenu,
active_routes: { path: 'projects/ci/pipeline_editor#show' },
item_id: :pipelines_editor
)
@@ -84,6 +91,7 @@ module Sidebars
::Sidebars::MenuItem.new(
title: _('Jobs'),
link: project_jobs_path(context.project),
+ super_sidebar_parent: ::Sidebars::Projects::SuperSidebarMenus::BuildMenu,
container_html_options: { class: 'shortcuts-builds' },
active_routes: { controller: :jobs },
item_id: :jobs
@@ -91,13 +99,10 @@ module Sidebars
end
def artifacts_menu_item
- unless Feature.enabled?(:artifacts_management_page, context.project)
- return ::Sidebars::NilMenuItem.new(item_id: :artifacts)
- end
-
::Sidebars::MenuItem.new(
title: _('Artifacts'),
link: project_artifacts_path(context.project),
+ super_sidebar_parent: ::Sidebars::Projects::SuperSidebarMenus::BuildMenu,
container_html_options: { class: 'shortcuts-builds' },
active_routes: { path: 'artifacts#index' },
item_id: :artifacts
@@ -106,8 +111,9 @@ module Sidebars
def pipeline_schedules_menu_item
::Sidebars::MenuItem.new(
- title: _('Schedules'),
+ title: context.is_super_sidebar ? _('Pipeline schedules') : _('Schedules'),
link: pipeline_schedules_path(context.project),
+ super_sidebar_parent: ::Sidebars::Projects::SuperSidebarMenus::BuildMenu,
container_html_options: { class: 'shortcuts-builds' },
active_routes: { controller: :pipeline_schedules },
item_id: :pipeline_schedules
diff --git a/lib/sidebars/projects/menus/confluence_menu.rb b/lib/sidebars/projects/menus/confluence_menu.rb
index 0fd42a57da3..43ef7ac73c4 100644
--- a/lib/sidebars/projects/menus/confluence_menu.rb
+++ b/lib/sidebars/projects/menus/confluence_menu.rb
@@ -42,6 +42,14 @@ module Sidebars
def active_routes
{ controller: :confluences }
end
+
+ override :serialize_as_menu_item_args
+ def serialize_as_menu_item_args
+ super.merge({
+ item_id: :confluence,
+ super_sidebar_parent: ::Sidebars::Projects::SuperSidebarMenus::PlanMenu
+ })
+ end
end
end
end
diff --git a/lib/sidebars/projects/menus/deployments_menu.rb b/lib/sidebars/projects/menus/deployments_menu.rb
index 4d4e65e9795..19612fcee85 100644
--- a/lib/sidebars/projects/menus/deployments_menu.rb
+++ b/lib/sidebars/projects/menus/deployments_menu.rb
@@ -34,6 +34,11 @@ module Sidebars
'deployments'
end
+ override :serialize_as_menu_item_args
+ def serialize_as_menu_item_args
+ nil
+ end
+
private
def feature_flags_menu_item
@@ -42,8 +47,9 @@ module Sidebars
end
::Sidebars::MenuItem.new(
- title: _('Feature Flags'),
+ title: s_('FeatureFlags|Feature flags'),
link: project_feature_flags_path(context.project),
+ super_sidebar_parent: ::Sidebars::Projects::SuperSidebarMenus::BuildMenu,
active_routes: { controller: :feature_flags },
container_html_options: { class: 'shortcuts-feature-flags' },
item_id: :feature_flags
@@ -58,6 +64,7 @@ module Sidebars
::Sidebars::MenuItem.new(
title: _('Environments'),
link: project_environments_path(context.project),
+ super_sidebar_parent: ::Sidebars::Projects::SuperSidebarMenus::BuildMenu,
active_routes: { controller: :environments },
container_html_options: { class: 'shortcuts-environments' },
item_id: :environments
@@ -73,6 +80,7 @@ module Sidebars
::Sidebars::MenuItem.new(
title: _('Releases'),
link: project_releases_path(context.project),
+ super_sidebar_parent: ::Sidebars::Projects::SuperSidebarMenus::BuildMenu,
item_id: :releases,
active_routes: { controller: :releases },
container_html_options: { class: 'shortcuts-deployments-releases' }
@@ -87,6 +95,7 @@ module Sidebars
::Sidebars::MenuItem.new(
title: _('Pages'),
link: project_pages_path(context.project),
+ super_sidebar_parent: Sidebars::Projects::SuperSidebarMenus::OperationsMenu,
active_routes: { path: 'pages#show' },
item_id: :pages
)
diff --git a/lib/sidebars/projects/menus/external_issue_tracker_menu.rb b/lib/sidebars/projects/menus/external_issue_tracker_menu.rb
index 136d30f38c3..f088ccce9f5 100644
--- a/lib/sidebars/projects/menus/external_issue_tracker_menu.rb
+++ b/lib/sidebars/projects/menus/external_issue_tracker_menu.rb
@@ -48,6 +48,14 @@ module Sidebars
external_issue_tracker.present?
end
+ override :serialize_as_menu_item_args
+ def serialize_as_menu_item_args
+ super.merge({
+ item_id: :external_issue_tracker,
+ super_sidebar_parent: ::Sidebars::Projects::SuperSidebarMenus::PlanMenu
+ })
+ end
+
private
def external_issue_tracker
diff --git a/lib/sidebars/projects/menus/external_wiki_menu.rb b/lib/sidebars/projects/menus/external_wiki_menu.rb
index 825f0ca5e8b..1af9abc33ff 100644
--- a/lib/sidebars/projects/menus/external_wiki_menu.rb
+++ b/lib/sidebars/projects/menus/external_wiki_menu.rb
@@ -41,6 +41,14 @@ module Sidebars
external_wiki.present?
end
+ override :serialize_as_menu_item_args
+ def serialize_as_menu_item_args
+ super.merge({
+ item_id: :external_wiki,
+ super_sidebar_parent: ::Sidebars::Projects::SuperSidebarMenus::PlanMenu
+ })
+ end
+
private
def external_wiki
diff --git a/lib/sidebars/projects/menus/infrastructure_menu.rb b/lib/sidebars/projects/menus/infrastructure_menu.rb
index 04c9ab77729..b08845a37e6 100644
--- a/lib/sidebars/projects/menus/infrastructure_menu.rb
+++ b/lib/sidebars/projects/menus/infrastructure_menu.rb
@@ -9,8 +9,9 @@ module Sidebars
return false unless feature_enabled?
add_item(kubernetes_menu_item)
- add_item(terraform_menu_item)
+ add_item(terraform_states_menu_item)
add_item(google_cloud_menu_item)
+ add_item(aws_menu_item)
true
end
@@ -32,6 +33,11 @@ module Sidebars
'cloud-gear'
end
+ override :serialize_as_menu_item_args
+ def serialize_as_menu_item_args
+ nil
+ end
+
private
def feature_enabled?
@@ -39,13 +45,14 @@ module Sidebars
end
def kubernetes_menu_item
- unless can?(context.current_user, :read_cluster, context.project)
+ unless can?(context.current_user, :read_cluster_agent, context.project)
return ::Sidebars::NilMenuItem.new(item_id: :kubernetes)
end
::Sidebars::MenuItem.new(
title: _('Kubernetes clusters'),
link: project_clusters_path(context.project),
+ super_sidebar_parent: Sidebars::Projects::SuperSidebarMenus::OperationsMenu,
active_routes: { controller: [:cluster_agents, :clusters] },
container_html_options: { class: 'shortcuts-kubernetes' },
hint_html_options: kubernetes_hint_html_options,
@@ -66,16 +73,37 @@ module Sidebars
auto_devops_help_path: help_page_path('topics/autodevops/index.md') } }
end
- def terraform_menu_item
+ def terraform_states_menu_item
unless can?(context.current_user, :read_terraform_state, context.project)
- return ::Sidebars::NilMenuItem.new(item_id: :terraform)
+ return ::Sidebars::NilMenuItem.new(item_id: :terraform_states)
end
::Sidebars::MenuItem.new(
- title: _('Terraform'),
+ title: s_('Terraform|Terraform states'),
link: project_terraform_index_path(context.project),
+ super_sidebar_parent: Sidebars::Projects::SuperSidebarMenus::OperationsMenu,
active_routes: { controller: :terraform },
- item_id: :terraform
+ item_id: :terraform_states
+ )
+ end
+
+ def aws_menu_item
+ enabled_for_user = Feature.enabled?(:cloudseed_aws, context.current_user)
+ enabled_for_group = Feature.enabled?(:cloudseed_aws, context.project.group)
+ enabled_for_project = Feature.enabled?(:cloudseed_aws, context.project)
+ feature_is_enabled = enabled_for_user || enabled_for_group || enabled_for_project
+ user_has_permissions = can?(context.current_user, :admin_project_aws, context.project)
+
+ return ::Sidebars::NilMenuItem.new(item_id: :cloudseed_aws) unless feature_is_enabled && user_has_permissions
+
+ ::Sidebars::MenuItem.new(
+ title: _('AWS'),
+ link: project_aws_path(context.project),
+ super_sidebar_parent: Sidebars::Projects::SuperSidebarMenus::OperationsMenu,
+ item_id: :aws,
+ active_routes: { controller: %w[
+ projects/aws/configuration
+ ] }
)
end
@@ -86,13 +114,16 @@ module Sidebars
feature_is_enabled = enabled_for_user || enabled_for_group || enabled_for_project
user_has_permissions = can?(context.current_user, :admin_project_google_cloud, context.project)
- unless feature_is_enabled && user_has_permissions
+ google_oauth2_configured = google_oauth2_configured?
+
+ unless feature_is_enabled && user_has_permissions && google_oauth2_configured
return ::Sidebars::NilMenuItem.new(item_id: :incubation_5mp_google_cloud)
end
::Sidebars::MenuItem.new(
title: _('Google Cloud'),
link: project_google_cloud_configuration_path(context.project),
+ super_sidebar_parent: Sidebars::Projects::SuperSidebarMenus::OperationsMenu,
active_routes: { controller: %w[
projects/google_cloud/configuration
projects/google_cloud/service_accounts
@@ -103,6 +134,11 @@ module Sidebars
item_id: :google_cloud
)
end
+
+ def google_oauth2_configured?
+ config = Gitlab::Auth::OAuth::Provider.config_for('google_oauth2')
+ config&.present? && config.app_id.present? && config.app_secret.present?
+ end
end
end
end
diff --git a/lib/sidebars/projects/menus/invite_team_members_menu.rb b/lib/sidebars/projects/menus/invite_team_members_menu.rb
deleted file mode 100644
index 0db49f1e12a..00000000000
--- a/lib/sidebars/projects/menus/invite_team_members_menu.rb
+++ /dev/null
@@ -1,47 +0,0 @@
-# frozen_string_literal: true
-
-module Sidebars
- module Projects
- module Menus
- class InviteTeamMembersMenu < ::Sidebars::Menu
- override :title
- def title
- s_('InviteMember|Invite members')
- end
-
- override :render?
- def render?
- can?(context.current_user, :admin_project_member, context.project) && all_valid_members.size <= 1
- end
-
- override :menu_partial
- def menu_partial
- 'projects/invite_members_side_nav_link'
- end
-
- override :menu_partial_options
- def menu_partial_options
- {
- project: context.project,
- title: title,
- sidebar_menu: self
- }
- end
-
- override :extra_nav_link_html_options
- def extra_nav_link_html_options
- {
- 'data-test-id': 'side-nav-invite-members'
- }
- end
-
- private
-
- def all_valid_members
- MembersFinder.new(context.project, context.current_user)
- .execute(include_relations: [:inherited, :direct, :invited_groups])
- end
- end
- end
- end
-end
diff --git a/lib/sidebars/projects/menus/issues_menu.rb b/lib/sidebars/projects/menus/issues_menu.rb
index 51eea3d850d..dc40b84529f 100644
--- a/lib/sidebars/projects/menus/issues_menu.rb
+++ b/lib/sidebars/projects/menus/issues_menu.rb
@@ -57,7 +57,8 @@ module Sidebars
override :pill_count
def pill_count
strong_memoize(:pill_count) do
- context.project.open_issues_count(context.current_user)
+ count = context.project.open_issues_count(context.current_user)
+ format_cached_count(1000, count)
end
end
@@ -68,16 +69,31 @@ module Sidebars
}
end
+ override :serialize_as_menu_item_args
+ def serialize_as_menu_item_args
+ super.merge({
+ pill_count: pill_count,
+ has_pill: has_pill?,
+ super_sidebar_parent: ::Sidebars::Projects::SuperSidebarMenus::PlanMenu,
+ item_id: :project_issue_list
+ })
+ end
+
private
def show_issues_menu_items?
can?(context.current_user, :read_issue, context.project)
end
+ def multi_issue_boards?
+ context.project.multiple_issue_boards_available?
+ end
+
def list_menu_item
::Sidebars::MenuItem.new(
title: _('List'),
link: project_issues_path(context.project),
+ super_sidebar_parent: ::Sidebars::NilMenuItem,
active_routes: { path: 'projects/issues#index' },
container_html_options: { aria: { label: _('Issues') } },
item_id: :issue_list
@@ -85,12 +101,18 @@ module Sidebars
end
def boards_menu_item
- title = context.project.multiple_issue_boards_available? ? s_('IssueBoards|Boards') : s_('IssueBoards|Board')
+ title = if context.is_super_sidebar
+ multi_issue_boards? ? s_('Issue boards') : s_('Issue board')
+ else
+ multi_issue_boards? ? s_('IssueBoards|Boards') : s_('IssueBoards|Board')
+ end
::Sidebars::MenuItem.new(
title: title,
link: project_boards_path(context.project),
+ super_sidebar_parent: ::Sidebars::Projects::SuperSidebarMenus::PlanMenu,
active_routes: { controller: :boards },
+ container_html_options: { class: 'shortcuts-issue-boards' },
item_id: :boards
)
end
@@ -99,6 +121,7 @@ module Sidebars
::Sidebars::MenuItem.new(
title: _('Service Desk'),
link: service_desk_project_issues_path(context.project),
+ super_sidebar_parent: ::Sidebars::Projects::SuperSidebarMenus::MonitorMenu,
active_routes: { path: 'issues#service_desk' },
item_id: :service_desk
)
@@ -108,6 +131,7 @@ module Sidebars
::Sidebars::MenuItem.new(
title: _('Milestones'),
link: project_milestones_path(context.project),
+ super_sidebar_parent: ::Sidebars::Projects::SuperSidebarMenus::PlanMenu,
active_routes: { controller: :milestones },
item_id: :milestones
)
diff --git a/lib/sidebars/projects/menus/merge_requests_menu.rb b/lib/sidebars/projects/menus/merge_requests_menu.rb
index 3e543872d36..ae4fd6b02e7 100644
--- a/lib/sidebars/projects/menus/merge_requests_menu.rb
+++ b/lib/sidebars/projects/menus/merge_requests_menu.rb
@@ -46,7 +46,8 @@ module Sidebars
override :pill_count
def pill_count
- @pill_count ||= context.project.open_merge_requests_count
+ count = @pill_count ||= context.project.open_merge_requests_count
+ format_cached_count(1000, count)
end
override :pill_html_options
@@ -59,11 +60,21 @@ module Sidebars
override :active_routes
def active_routes
if context.project.issues_enabled?
- { controller: 'projects/merge_requests' }
+ { controller: ['projects/merge_requests', :conflicts] }
else
- { controller: ['projects/merge_requests', :milestones] }
+ { controller: ['projects/merge_requests', :milestones, :conflicts] }
end
end
+
+ override :serialize_as_menu_item_args
+ def serialize_as_menu_item_args
+ super.merge({
+ pill_count: pill_count,
+ has_pill: has_pill?,
+ super_sidebar_parent: ::Sidebars::Projects::SuperSidebarMenus::CodeMenu,
+ item_id: :project_merge_request_list
+ })
+ end
end
end
end
diff --git a/lib/sidebars/projects/menus/monitor_menu.rb b/lib/sidebars/projects/menus/monitor_menu.rb
index 7d1fa8b8fa7..f1fc9f70ef8 100644
--- a/lib/sidebars/projects/menus/monitor_menu.rb
+++ b/lib/sidebars/projects/menus/monitor_menu.rb
@@ -12,7 +12,6 @@ module Sidebars
add_item(error_tracking_menu_item)
add_item(alert_management_menu_item)
add_item(incidents_menu_item)
- add_item(airflow_dashboard_menu_item)
true
end
@@ -39,6 +38,11 @@ module Sidebars
{ controller: [:user, :gcp] }
end
+ override :serialize_as_menu_item_args
+ def serialize_as_menu_item_args
+ nil
+ end
+
private
def feature_enabled?
@@ -46,6 +50,8 @@ module Sidebars
end
def metrics_dashboard_menu_item
+ return ::Sidebars::NilMenuItem.new(item_id: :metrics) if Feature.enabled?(:remove_monitor_metrics)
+
unless can?(context.current_user, :metrics_dashboard, context.project)
return ::Sidebars::NilMenuItem.new(item_id: :metrics)
end
@@ -53,6 +59,7 @@ module Sidebars
::Sidebars::MenuItem.new(
title: _('Metrics'),
link: project_metrics_dashboard_path(context.project),
+ super_sidebar_parent: ::Sidebars::Projects::SuperSidebarMenus::MonitorMenu,
active_routes: { path: 'metrics_dashboard#show' },
container_html_options: { class: 'shortcuts-metrics' },
item_id: :metrics
@@ -67,6 +74,7 @@ module Sidebars
::Sidebars::MenuItem.new(
title: _('Error Tracking'),
link: project_error_tracking_index_path(context.project),
+ super_sidebar_parent: ::Sidebars::Projects::SuperSidebarMenus::MonitorMenu,
active_routes: { controller: :error_tracking },
item_id: :error_tracking
)
@@ -80,6 +88,7 @@ module Sidebars
::Sidebars::MenuItem.new(
title: _('Alerts'),
link: project_alert_management_index_path(context.project),
+ super_sidebar_parent: ::Sidebars::Projects::SuperSidebarMenus::MonitorMenu,
active_routes: { controller: :alert_management },
item_id: :alert_management
)
@@ -93,24 +102,11 @@ module Sidebars
::Sidebars::MenuItem.new(
title: _('Incidents'),
link: project_incidents_path(context.project),
+ super_sidebar_parent: ::Sidebars::Projects::SuperSidebarMenus::MonitorMenu,
active_routes: { controller: [:incidents, :incident_management] },
item_id: :incidents
)
end
-
- def airflow_dashboard_menu_item
- unless can?(context.current_user, :read_airflow_dags, context.project) &&
- Feature.enabled?(:airflow_dags, context.project)
- return ::Sidebars::NilMenuItem.new(item_id: :airflow)
- end
-
- ::Sidebars::MenuItem.new(
- title: _('Airflow'),
- link: project_airflow_dags_path(context.project),
- active_routes: { path: 'airflow/dags#show' },
- item_id: :airflow_dags
- )
- end
end
end
end
diff --git a/lib/sidebars/projects/menus/packages_registries_menu.rb b/lib/sidebars/projects/menus/packages_registries_menu.rb
index fc7c564574a..31a1aa56ab5 100644
--- a/lib/sidebars/projects/menus/packages_registries_menu.rb
+++ b/lib/sidebars/projects/menus/packages_registries_menu.rb
@@ -10,6 +10,7 @@ module Sidebars
add_item(container_registry_menu_item)
add_item(infrastructure_registry_menu_item)
add_item(harbor_registry_menu_item)
+ add_item(model_experiments_menu_item)
true
end
@@ -23,6 +24,11 @@ module Sidebars
'package'
end
+ override :serialize_as_menu_item_args
+ def serialize_as_menu_item_args
+ nil
+ end
+
private
def packages_registry_menu_item
@@ -33,6 +39,7 @@ module Sidebars
::Sidebars::MenuItem.new(
title: _('Package Registry'),
link: project_packages_path(context.project),
+ super_sidebar_parent: Sidebars::Projects::SuperSidebarMenus::OperationsMenu,
active_routes: { controller: :packages },
item_id: :packages_registry,
container_html_options: { class: 'shortcuts-container-registry' }
@@ -47,6 +54,7 @@ module Sidebars
::Sidebars::MenuItem.new(
title: _('Container Registry'),
link: project_container_registry_index_path(context.project),
+ super_sidebar_parent: Sidebars::Projects::SuperSidebarMenus::OperationsMenu,
active_routes: { controller: 'projects/registry/repositories' },
item_id: :container_registry
)
@@ -58,8 +66,9 @@ module Sidebars
end
::Sidebars::MenuItem.new(
- title: _('Infrastructure Registry'),
+ title: _('Terraform modules'),
link: project_infrastructure_registry_index_path(context.project),
+ super_sidebar_parent: Sidebars::Projects::SuperSidebarMenus::OperationsMenu,
active_routes: { controller: :infrastructure_registry },
item_id: :infrastructure_registry
)
@@ -75,11 +84,26 @@ module Sidebars
::Sidebars::MenuItem.new(
title: _('Harbor Registry'),
link: project_harbor_repositories_path(context.project),
- active_routes: { controller: :harbor_registry },
+ super_sidebar_parent: Sidebars::Projects::SuperSidebarMenus::OperationsMenu,
+ active_routes: { controller: 'projects/harbor/repositories' },
item_id: :harbor_registry
)
end
+ def model_experiments_menu_item
+ if Feature.disabled?(:ml_experiment_tracking, context.project)
+ return ::Sidebars::NilMenuItem.new(item_id: :model_experiments)
+ end
+
+ ::Sidebars::MenuItem.new(
+ title: _('Model experiments'),
+ link: project_ml_experiments_path(context.project),
+ super_sidebar_parent: Sidebars::Projects::SuperSidebarMenus::AnalyzeMenu,
+ active_routes: { controller: %w[projects/ml/experiments projects/ml/candidates] },
+ item_id: :model_experiments
+ )
+ end
+
def packages_registry_disabled?
!::Gitlab.config.packages.enabled ||
!can?(context.current_user, :read_package, context.project&.packages_policy_subject)
diff --git a/lib/sidebars/projects/menus/project_information_menu.rb b/lib/sidebars/projects/menus/project_information_menu.rb
index 44b94ee3522..6ab7e00dad3 100644
--- a/lib/sidebars/projects/menus/project_information_menu.rb
+++ b/lib/sidebars/projects/menus/project_information_menu.rb
@@ -33,12 +33,18 @@ module Sidebars
'project'
end
+ override :serialize_as_menu_item_args
+ def serialize_as_menu_item_args
+ nil
+ end
+
private
def activity_menu_item
::Sidebars::MenuItem.new(
title: _('Activity'),
link: activity_project_path(context.project),
+ super_sidebar_parent: ::Sidebars::Projects::SuperSidebarMenus::ManageMenu,
active_routes: { path: 'projects#activity' },
item_id: :activity,
container_html_options: { class: 'shortcuts-project-activity' }
@@ -53,6 +59,7 @@ module Sidebars
::Sidebars::MenuItem.new(
title: _('Labels'),
link: project_labels_path(context.project),
+ super_sidebar_parent: ::Sidebars::Projects::SuperSidebarMenus::ManageMenu,
active_routes: { controller: :labels },
item_id: :labels
)
@@ -66,6 +73,7 @@ module Sidebars
::Sidebars::MenuItem.new(
title: _('Members'),
link: project_project_members_path(context.project),
+ super_sidebar_parent: ::Sidebars::Projects::SuperSidebarMenus::ManageMenu,
active_routes: { controller: :project_members },
item_id: :members,
container_html_options: {
diff --git a/lib/sidebars/projects/menus/repository_menu.rb b/lib/sidebars/projects/menus/repository_menu.rb
index ec91ae741fe..22f7b553884 100644
--- a/lib/sidebars/projects/menus/repository_menu.rb
+++ b/lib/sidebars/projects/menus/repository_menu.rb
@@ -44,13 +44,20 @@ module Sidebars
'doc-text'
end
+ override :serialize_as_menu_item_args
+ def serialize_as_menu_item_args
+ nil
+ end
+
private
def files_menu_item
::Sidebars::MenuItem.new(
- title: _('Files'),
+ title: context.is_super_sidebar ? _('Repository') : _('Files'),
link: project_tree_path(context.project, context.current_ref),
+ super_sidebar_parent: ::Sidebars::Projects::SuperSidebarMenus::CodeMenu,
active_routes: { controller: %w[tree blob blame edit_tree new_tree find_file] },
+ container_html_options: { class: 'shortcuts-tree' },
item_id: :files
)
end
@@ -61,9 +68,10 @@ module Sidebars
::Sidebars::MenuItem.new(
title: _('Commits'),
link: link,
+ super_sidebar_parent: ::Sidebars::Projects::SuperSidebarMenus::CodeMenu,
active_routes: { controller: %w(commit commits) },
item_id: :commits,
- container_html_options: { id: 'js-onboarding-commits-link' }
+ container_html_options: { id: 'js-onboarding-commits-link', class: 'shortcuts-commits' }
)
end
@@ -71,6 +79,7 @@ module Sidebars
::Sidebars::MenuItem.new(
title: _('Branches'),
link: project_branches_path(context.project),
+ super_sidebar_parent: ::Sidebars::Projects::SuperSidebarMenus::CodeMenu,
active_routes: { controller: :branches },
item_id: :branches,
container_html_options: { id: 'js-onboarding-branches-link' }
@@ -81,6 +90,7 @@ module Sidebars
::Sidebars::MenuItem.new(
title: _('Tags'),
link: project_tags_path(context.project),
+ super_sidebar_parent: ::Sidebars::Projects::SuperSidebarMenus::CodeMenu,
item_id: :tags,
active_routes: { controller: :tags }
)
@@ -92,8 +102,9 @@ module Sidebars
link = project_graph_path(context.project, context.current_ref, ref_type: ref_type_from_context(context))
::Sidebars::MenuItem.new(
- title: _('Contributors'),
+ title: _('Contributor statistics'),
link: link,
+ super_sidebar_parent: ::Sidebars::Projects::SuperSidebarMenus::AnalyzeMenu,
active_routes: { path: 'graphs#show' },
item_id: :contributors
)
@@ -103,16 +114,19 @@ module Sidebars
link = project_network_path(context.project, context.current_ref, ref_type: ref_type_from_context(context))
::Sidebars::MenuItem.new(
- title: _('Graph'),
+ title: context.is_super_sidebar ? _('Repository graph') : _('Graph'),
link: link,
+ super_sidebar_parent: ::Sidebars::Projects::SuperSidebarMenus::CodeMenu,
active_routes: { controller: :network },
+ container_html_options: { class: 'shortcuts-network' },
item_id: :graphs
)
end
def compare_menu_item
::Sidebars::MenuItem.new(
- title: _('Compare'),
+ title: _('Compare revisions'),
+ super_sidebar_parent: ::Sidebars::Projects::SuperSidebarMenus::CodeMenu,
link: project_compare_index_path(context.project, from: context.project.repository.root_ref, to: context.current_ref),
active_routes: { controller: :compare },
item_id: :compare
diff --git a/lib/sidebars/projects/menus/scope_menu.rb b/lib/sidebars/projects/menus/scope_menu.rb
index 35502c7ea09..7e2286633e5 100644
--- a/lib/sidebars/projects/menus/scope_menu.rb
+++ b/lib/sidebars/projects/menus/scope_menu.rb
@@ -39,6 +39,16 @@ module Sidebars
def render?
true
end
+
+ override :serialize_as_menu_item_args
+ def serialize_as_menu_item_args
+ super.merge({
+ title: _('Project overview'),
+ sprite_icon: 'project',
+ super_sidebar_parent: ::Sidebars::StaticMenu,
+ item_id: :project_overview
+ })
+ end
end
end
end
diff --git a/lib/sidebars/projects/menus/security_compliance_menu.rb b/lib/sidebars/projects/menus/security_compliance_menu.rb
index 9367514cdca..0f009bff12a 100644
--- a/lib/sidebars/projects/menus/security_compliance_menu.rb
+++ b/lib/sidebars/projects/menus/security_compliance_menu.rb
@@ -17,7 +17,7 @@ module Sidebars
override :title
def title
- _('Security & Compliance')
+ _('Security and Compliance')
end
override :sprite_icon
@@ -25,6 +25,11 @@ module Sidebars
'shield'
end
+ override :serialize_as_menu_item_args
+ def serialize_as_menu_item_args
+ nil
+ end
+
private
def configuration_menu_item
@@ -33,8 +38,9 @@ module Sidebars
end
::Sidebars::MenuItem.new(
- title: _('Configuration'),
+ title: _('Security configuration'),
link: project_security_configuration_path(context.project),
+ super_sidebar_parent: ::Sidebars::Projects::SuperSidebarMenus::SecureMenu,
active_routes: { path: configuration_menu_item_paths },
item_id: :configuration
)
diff --git a/lib/sidebars/projects/menus/settings_menu.rb b/lib/sidebars/projects/menus/settings_menu.rb
index eea32d8b626..9ed142f7f60 100644
--- a/lib/sidebars/projects/menus/settings_menu.rb
+++ b/lib/sidebars/projects/menus/settings_menu.rb
@@ -44,6 +44,16 @@ module Sidebars
'settings'
end
+ override :pick_into_super_sidebar?
+ def pick_into_super_sidebar?
+ true
+ end
+
+ override :separated?
+ def separated?
+ true
+ end
+
private
def general_menu_item
@@ -163,7 +173,7 @@ module Sidebars
title: _('Merge requests'),
link: project_settings_merge_requests_path(context.project),
active_routes: { path: 'projects/settings/merge_requests#show' },
- item_id: :merge_requests
+ item_id: context.is_super_sidebar ? :merge_request_settings : :merge_requests
)
end
end
diff --git a/lib/sidebars/projects/menus/snippets_menu.rb b/lib/sidebars/projects/menus/snippets_menu.rb
index 060341b3c51..ba458f9f5e9 100644
--- a/lib/sidebars/projects/menus/snippets_menu.rb
+++ b/lib/sidebars/projects/menus/snippets_menu.rb
@@ -35,6 +35,14 @@ module Sidebars
def active_routes
{ controller: :snippets }
end
+
+ override :serialize_as_menu_item_args
+ def serialize_as_menu_item_args
+ super.deep_merge({
+ super_sidebar_parent: ::Sidebars::Projects::SuperSidebarMenus::CodeMenu,
+ item_id: :project_snippets
+ })
+ end
end
end
end
diff --git a/lib/sidebars/projects/menus/wiki_menu.rb b/lib/sidebars/projects/menus/wiki_menu.rb
index 3980b193fd1..d800dae8be3 100644
--- a/lib/sidebars/projects/menus/wiki_menu.rb
+++ b/lib/sidebars/projects/menus/wiki_menu.rb
@@ -35,6 +35,14 @@ module Sidebars
def active_routes
{ controller: :wikis }
end
+
+ override :serialize_as_menu_item_args
+ def serialize_as_menu_item_args
+ super.merge({
+ super_sidebar_parent: ::Sidebars::Projects::SuperSidebarMenus::PlanMenu,
+ item_id: :project_wiki
+ })
+ end
end
end
end
diff --git a/lib/sidebars/projects/panel.rb b/lib/sidebars/projects/panel.rb
index 9d0f5eb87bd..5d8bc18ac88 100644
--- a/lib/sidebars/projects/panel.rb
+++ b/lib/sidebars/projects/panel.rb
@@ -34,14 +34,6 @@ module Sidebars
add_wiki_menus
add_menu(Sidebars::Projects::Menus::SnippetsMenu.new(context))
add_menu(Sidebars::Projects::Menus::SettingsMenu.new(context))
- add_invite_members_menu
- end
-
- def add_invite_members_menu
- experiment(:invite_members_in_side_nav, group: context.project.group) do |e|
- e.control {}
- e.candidate { add_menu(Sidebars::Projects::Menus::InviteTeamMembersMenu.new(context)) }
- end
end
def add_wiki_menus
diff --git a/lib/sidebars/projects/super_sidebar_menus/analyze_menu.rb b/lib/sidebars/projects/super_sidebar_menus/analyze_menu.rb
new file mode 100644
index 00000000000..58b231a269c
--- /dev/null
+++ b/lib/sidebars/projects/super_sidebar_menus/analyze_menu.rb
@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module Projects
+ module SuperSidebarMenus
+ class AnalyzeMenu < ::Sidebars::Menu
+ override :title
+ def title
+ s_('Navigation|Analyze')
+ end
+
+ override :sprite_icon
+ def sprite_icon
+ 'chart'
+ end
+
+ override :configure_menu_items
+ def configure_menu_items
+ [
+ :dashboards_analytics,
+ :cycle_analytics,
+ :contributors,
+ :ci_cd_analytics,
+ :repository_analytics,
+ :code_review,
+ :merge_request_analytics,
+ :issues,
+ :insights,
+ :model_experiments
+ ].each { |id| add_item(::Sidebars::NilMenuItem.new(item_id: id)) }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/sidebars/projects/super_sidebar_menus/build_menu.rb b/lib/sidebars/projects/super_sidebar_menus/build_menu.rb
new file mode 100644
index 00000000000..30603e1deeb
--- /dev/null
+++ b/lib/sidebars/projects/super_sidebar_menus/build_menu.rb
@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module Projects
+ module SuperSidebarMenus
+ class BuildMenu < ::Sidebars::Menu
+ override :title
+ def title
+ s_('Navigation|Build')
+ end
+
+ override :sprite_icon
+ def sprite_icon
+ 'rocket'
+ end
+
+ override :configure_menu_items
+ def configure_menu_items
+ [
+ :pipelines,
+ :jobs,
+ :pipelines_editor,
+ :releases,
+ :environments,
+ :pipeline_schedules,
+ :feature_flags,
+ :test_cases,
+ :artifacts
+ ].each { |id| add_item(::Sidebars::NilMenuItem.new(item_id: id)) }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/sidebars/projects/super_sidebar_menus/code_menu.rb b/lib/sidebars/projects/super_sidebar_menus/code_menu.rb
new file mode 100644
index 00000000000..89acb92e3b8
--- /dev/null
+++ b/lib/sidebars/projects/super_sidebar_menus/code_menu.rb
@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module Projects
+ module SuperSidebarMenus
+ class CodeMenu < ::Sidebars::Menu
+ override :title
+ def title
+ s_('Navigation|Code')
+ end
+
+ override :sprite_icon
+ def sprite_icon
+ 'code'
+ end
+
+ override :configure_menu_items
+ def configure_menu_items
+ [
+ :project_merge_request_list,
+ :files,
+ :branches,
+ :commits,
+ :tags,
+ :graphs,
+ :compare,
+ :project_snippets,
+ :file_locks
+ ].each { |id| add_item(::Sidebars::NilMenuItem.new(item_id: id)) }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/sidebars/projects/super_sidebar_menus/manage_menu.rb b/lib/sidebars/projects/super_sidebar_menus/manage_menu.rb
new file mode 100644
index 00000000000..f36588628a9
--- /dev/null
+++ b/lib/sidebars/projects/super_sidebar_menus/manage_menu.rb
@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module Projects
+ module SuperSidebarMenus
+ class ManageMenu < ::Sidebars::Menu
+ override :title
+ def title
+ s_('Navigation|Manage')
+ end
+
+ override :sprite_icon
+ def sprite_icon
+ 'users'
+ end
+
+ override :configure_menu_items
+ def configure_menu_items
+ [
+ :activity,
+ :members,
+ :labels
+ ].each { |id| add_item(::Sidebars::NilMenuItem.new(item_id: id)) }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/sidebars/projects/super_sidebar_menus/monitor_menu.rb b/lib/sidebars/projects/super_sidebar_menus/monitor_menu.rb
new file mode 100644
index 00000000000..fb56f6f3792
--- /dev/null
+++ b/lib/sidebars/projects/super_sidebar_menus/monitor_menu.rb
@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module Projects
+ module SuperSidebarMenus
+ class MonitorMenu < ::Sidebars::Menu
+ override :title
+ def title
+ s_('Navigation|Monitor')
+ end
+
+ override :sprite_icon
+ def sprite_icon
+ 'monitor'
+ end
+
+ override :configure_menu_items
+ def configure_menu_items
+ [
+ :metrics,
+ :error_tracking,
+ :alert_management,
+ :incidents,
+ :on_call_schedules,
+ :escalation_policies,
+ :service_desk
+ ].each { |id| add_item(::Sidebars::NilMenuItem.new(item_id: id)) }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/sidebars/projects/super_sidebar_menus/operations_menu.rb b/lib/sidebars/projects/super_sidebar_menus/operations_menu.rb
new file mode 100644
index 00000000000..64cf4aee9c5
--- /dev/null
+++ b/lib/sidebars/projects/super_sidebar_menus/operations_menu.rb
@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module Projects
+ module SuperSidebarMenus
+ class OperationsMenu < ::Sidebars::Menu
+ override :title
+ def title
+ s_('Navigation|Operate')
+ end
+
+ override :sprite_icon
+ def sprite_icon
+ 'deployments'
+ end
+
+ override :configure_menu_items
+ def configure_menu_items
+ [
+ :packages_registry,
+ :container_registry,
+ :kubernetes,
+ :terraform_states,
+ :infrastructure_registry,
+ :google_cloud,
+ :aws
+ ].each { |id| add_item(::Sidebars::NilMenuItem.new(item_id: id)) }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/sidebars/projects/super_sidebar_menus/plan_menu.rb b/lib/sidebars/projects/super_sidebar_menus/plan_menu.rb
new file mode 100644
index 00000000000..dc35b37fe70
--- /dev/null
+++ b/lib/sidebars/projects/super_sidebar_menus/plan_menu.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module Projects
+ module SuperSidebarMenus
+ class PlanMenu < ::Sidebars::Menu
+ override :title
+ def title
+ s_('Navigation|Plan')
+ end
+
+ override :sprite_icon
+ def sprite_icon
+ 'planning'
+ end
+
+ override :configure_menu_items
+ def configure_menu_items
+ [
+ :project_issue_list,
+ :boards,
+ :milestones,
+ :iterations,
+ :project_wiki,
+ :requirements
+ ].each { |id| add_item(::Sidebars::NilMenuItem.new(item_id: id)) }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/sidebars/projects/super_sidebar_menus/secure_menu.rb b/lib/sidebars/projects/super_sidebar_menus/secure_menu.rb
new file mode 100644
index 00000000000..2ca53ba0ce9
--- /dev/null
+++ b/lib/sidebars/projects/super_sidebar_menus/secure_menu.rb
@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module Projects
+ module SuperSidebarMenus
+ class SecureMenu < ::Sidebars::Menu
+ override :title
+ def title
+ s_('Navigation|Secure')
+ end
+
+ override :sprite_icon
+ def sprite_icon
+ 'shield'
+ end
+
+ override :configure_menu_items
+ def configure_menu_items
+ [
+ :discover_project_security,
+ :dashboard,
+ :vulnerability_report,
+ :dependency_list,
+ :license_compliance,
+ :audit_events,
+ :scan_policies,
+ :on_demand_scans,
+ :configuration
+ ].each { |id| add_item(::Sidebars::NilMenuItem.new(item_id: id)) }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/sidebars/projects/super_sidebar_panel.rb b/lib/sidebars/projects/super_sidebar_panel.rb
new file mode 100644
index 00000000000..640666fd968
--- /dev/null
+++ b/lib/sidebars/projects/super_sidebar_panel.rb
@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module Projects
+ class SuperSidebarPanel < ::Sidebars::Projects::Panel
+ include ::Sidebars::Concerns::SuperSidebarPanel
+ extend ::Gitlab::Utils::Override
+
+ override :configure_menus
+ def configure_menus
+ super
+ old_menus = @menus
+ @menus = []
+
+ add_menu(Sidebars::StaticMenu.new(context))
+ add_menu(Sidebars::Projects::SuperSidebarMenus::ManageMenu.new(context))
+ add_menu(Sidebars::Projects::SuperSidebarMenus::PlanMenu.new(context))
+ add_menu(Sidebars::Projects::SuperSidebarMenus::CodeMenu.new(context))
+ add_menu(Sidebars::Projects::SuperSidebarMenus::BuildMenu.new(context))
+ add_menu(Sidebars::Projects::SuperSidebarMenus::SecureMenu.new(context))
+ add_menu(Sidebars::Projects::SuperSidebarMenus::OperationsMenu.new(context))
+ add_menu(Sidebars::Projects::SuperSidebarMenus::MonitorMenu.new(context))
+ add_menu(Sidebars::Projects::SuperSidebarMenus::AnalyzeMenu.new(context))
+
+ # Pick old menus, will be obsolete once everything is in their own
+ # super sidebar menu
+ pick_from_old_menus(old_menus)
+
+ insert_menu_before(
+ Sidebars::Projects::Menus::SettingsMenu,
+ Sidebars::UncategorizedMenu.new(context)
+ )
+
+ transform_old_menus(@menus, @scope_menu, *old_menus)
+ end
+
+ override :super_sidebar_context_header
+ def super_sidebar_context_header
+ {
+ title: context.project.name,
+ avatar: context.project.avatar_url,
+ id: context.project.id
+ }
+ end
+ end
+ end
+end
diff --git a/lib/sidebars/search/panel.rb b/lib/sidebars/search/panel.rb
new file mode 100644
index 00000000000..83079fa8e72
--- /dev/null
+++ b/lib/sidebars/search/panel.rb
@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module Search
+ class Panel < ::Sidebars::Panel
+ override :aria_label
+ def aria_label
+ _('Search results')
+ end
+
+ override :super_sidebar_context_header
+ def super_sidebar_context_header
+ @super_sidebar_context_header ||= {
+ title: aria_label,
+ icon: 'search-results'
+ }
+ end
+ end
+ end
+end
diff --git a/lib/sidebars/static_menu.rb b/lib/sidebars/static_menu.rb
new file mode 100644
index 00000000000..b7ba69b1717
--- /dev/null
+++ b/lib/sidebars/static_menu.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Sidebars
+ # This is a special menu which does not serialize as
+ # a section and instead hoists all of menu items
+ # to be top-level items
+ class StaticMenu < ::Sidebars::Menu
+ override :serialize_for_super_sidebar
+ def serialize_for_super_sidebar
+ serialize_items_for_super_sidebar
+ end
+ end
+end
diff --git a/lib/sidebars/uncategorized_menu.rb b/lib/sidebars/uncategorized_menu.rb
new file mode 100644
index 00000000000..dc9ed8308fa
--- /dev/null
+++ b/lib/sidebars/uncategorized_menu.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Sidebars
+ # This Menu is a temporary help while we implement the new menu
+ # categories for everything. Once every Menu Item is categorized,
+ # we can remove this. This should be done before the Super Sidebar
+ # moves out of Alpha.
+ class UncategorizedMenu < ::Sidebars::Menu
+ override :title
+ def title
+ _('Uncategorized')
+ end
+
+ override :sprite_icon
+ def sprite_icon
+ 'question'
+ end
+ end
+end
diff --git a/lib/sidebars/user_profile/base_menu.rb b/lib/sidebars/user_profile/base_menu.rb
new file mode 100644
index 00000000000..5594d7c3c65
--- /dev/null
+++ b/lib/sidebars/user_profile/base_menu.rb
@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module UserProfile
+ class BaseMenu < ::Sidebars::Menu
+ override :render?
+ def render?
+ can?(context.current_user, :read_user_profile, context.container)
+ end
+ end
+ end
+end
diff --git a/lib/sidebars/user_profile/menus/activity_menu.rb b/lib/sidebars/user_profile/menus/activity_menu.rb
new file mode 100644
index 00000000000..8de51f9b05b
--- /dev/null
+++ b/lib/sidebars/user_profile/menus/activity_menu.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module UserProfile
+ module Menus
+ class ActivityMenu < ::Sidebars::UserProfile::BaseMenu
+ override :link
+ def link
+ user_activity_path(context.container)
+ end
+
+ override :title
+ def title
+ s_('UserProfile|Activity')
+ end
+
+ override :sprite_icon
+ def sprite_icon
+ 'history'
+ end
+
+ override :active_routes
+ def active_routes
+ { path: 'users#activity' }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/sidebars/user_profile/menus/contributed_projects_menu.rb b/lib/sidebars/user_profile/menus/contributed_projects_menu.rb
new file mode 100644
index 00000000000..c4f8c364dc4
--- /dev/null
+++ b/lib/sidebars/user_profile/menus/contributed_projects_menu.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module UserProfile
+ module Menus
+ class ContributedProjectsMenu < ::Sidebars::UserProfile::BaseMenu
+ override :link
+ def link
+ user_contributed_projects_path(context.container)
+ end
+
+ override :title
+ def title
+ s_('UserProfile|Contributed projects')
+ end
+
+ override :sprite_icon
+ def sprite_icon
+ 'project'
+ end
+
+ override :active_routes
+ def active_routes
+ { path: 'users#contributed' }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/sidebars/user_profile/menus/followers_menu.rb b/lib/sidebars/user_profile/menus/followers_menu.rb
new file mode 100644
index 00000000000..e27c9ec7f8f
--- /dev/null
+++ b/lib/sidebars/user_profile/menus/followers_menu.rb
@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module UserProfile
+ module Menus
+ class FollowersMenu < ::Sidebars::UserProfile::BaseMenu
+ include Gitlab::Utils::StrongMemoize
+
+ override :link
+ def link
+ user_followers_path(context.container)
+ end
+
+ override :title
+ def title
+ s_('UserProfile|Followers')
+ end
+
+ override :sprite_icon
+ def sprite_icon
+ 'users'
+ end
+
+ override :active_routes
+ def active_routes
+ { path: 'users#followers' }
+ end
+
+ override :has_pill?
+ def has_pill?
+ context.container.followers.any?
+ end
+ strong_memoize_attr :has_pill?
+
+ override :pill_count
+ def pill_count
+ context.container.followers.count
+ end
+ strong_memoize_attr :pill_count
+ end
+ end
+ end
+end
diff --git a/lib/sidebars/user_profile/menus/following_menu.rb b/lib/sidebars/user_profile/menus/following_menu.rb
new file mode 100644
index 00000000000..b6b23259170
--- /dev/null
+++ b/lib/sidebars/user_profile/menus/following_menu.rb
@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module UserProfile
+ module Menus
+ class FollowingMenu < ::Sidebars::UserProfile::BaseMenu
+ include Gitlab::Utils::StrongMemoize
+
+ override :link
+ def link
+ user_following_path(context.container)
+ end
+
+ override :title
+ def title
+ s_('UserProfile|Following')
+ end
+
+ override :sprite_icon
+ def sprite_icon
+ 'users'
+ end
+
+ override :active_routes
+ def active_routes
+ { path: 'users#following' }
+ end
+
+ override :has_pill?
+ def has_pill?
+ context.container.followees.any?
+ end
+ strong_memoize_attr :has_pill?
+
+ override :pill_count
+ def pill_count
+ context.container.followees.count
+ end
+ strong_memoize_attr :pill_count
+ end
+ end
+ end
+end
diff --git a/lib/sidebars/user_profile/menus/groups_menu.rb b/lib/sidebars/user_profile/menus/groups_menu.rb
new file mode 100644
index 00000000000..0b5d23e0946
--- /dev/null
+++ b/lib/sidebars/user_profile/menus/groups_menu.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module UserProfile
+ module Menus
+ class GroupsMenu < ::Sidebars::UserProfile::BaseMenu
+ override :link
+ def link
+ user_groups_path(context.container)
+ end
+
+ override :title
+ def title
+ s_('UserProfile|Groups')
+ end
+
+ override :sprite_icon
+ def sprite_icon
+ 'group'
+ end
+
+ override :active_routes
+ def active_routes
+ { path: 'users#groups' }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/sidebars/user_profile/menus/overview_menu.rb b/lib/sidebars/user_profile/menus/overview_menu.rb
new file mode 100644
index 00000000000..e9fa515e9bc
--- /dev/null
+++ b/lib/sidebars/user_profile/menus/overview_menu.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module UserProfile
+ module Menus
+ class OverviewMenu < ::Sidebars::UserProfile::BaseMenu
+ override :link
+ def link
+ user_path(context.container)
+ end
+
+ override :title
+ def title
+ s_('UserProfile|Overview')
+ end
+
+ override :sprite_icon
+ def sprite_icon
+ 'overview'
+ end
+
+ override :active_routes
+ def active_routes
+ { path: 'users#show' }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/sidebars/user_profile/menus/personal_projects_menu.rb b/lib/sidebars/user_profile/menus/personal_projects_menu.rb
new file mode 100644
index 00000000000..21ff4c07854
--- /dev/null
+++ b/lib/sidebars/user_profile/menus/personal_projects_menu.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module UserProfile
+ module Menus
+ class PersonalProjectsMenu < ::Sidebars::UserProfile::BaseMenu
+ override :link
+ def link
+ user_projects_path(context.container)
+ end
+
+ override :title
+ def title
+ s_('UserProfile|Personal projects')
+ end
+
+ override :sprite_icon
+ def sprite_icon
+ 'project'
+ end
+
+ override :active_routes
+ def active_routes
+ { path: 'users#projects' }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/sidebars/user_profile/menus/snippets_menu.rb b/lib/sidebars/user_profile/menus/snippets_menu.rb
new file mode 100644
index 00000000000..e93c9ac5cd3
--- /dev/null
+++ b/lib/sidebars/user_profile/menus/snippets_menu.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module UserProfile
+ module Menus
+ class SnippetsMenu < ::Sidebars::UserProfile::BaseMenu
+ override :link
+ def link
+ user_snippets_path(context.container)
+ end
+
+ override :title
+ def title
+ s_('UserProfile|Snippets')
+ end
+
+ override :sprite_icon
+ def sprite_icon
+ 'snippet'
+ end
+
+ override :active_routes
+ def active_routes
+ { path: 'users#snippets' }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/sidebars/user_profile/menus/starred_projects_menu.rb b/lib/sidebars/user_profile/menus/starred_projects_menu.rb
new file mode 100644
index 00000000000..02d7e67850f
--- /dev/null
+++ b/lib/sidebars/user_profile/menus/starred_projects_menu.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module UserProfile
+ module Menus
+ class StarredProjectsMenu < ::Sidebars::UserProfile::BaseMenu
+ override :link
+ def link
+ user_starred_projects_path(context.container)
+ end
+
+ override :title
+ def title
+ s_('UserProfile|Starred projects')
+ end
+
+ override :sprite_icon
+ def sprite_icon
+ 'star-o'
+ end
+
+ override :active_routes
+ def active_routes
+ { path: 'users#starred' }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/sidebars/user_profile/panel.rb b/lib/sidebars/user_profile/panel.rb
new file mode 100644
index 00000000000..9a595fdf64c
--- /dev/null
+++ b/lib/sidebars/user_profile/panel.rb
@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module UserProfile
+ class Panel < ::Sidebars::Panel
+ include UsersHelper
+
+ override :configure_menus
+ def configure_menus
+ add_menus
+ end
+
+ override :aria_label
+ def aria_label
+ s_('UserProfile|User profile navigation')
+ end
+
+ override :super_sidebar_context_header
+ def super_sidebar_context_header
+ @super_sidebar_context_header ||= {
+ title: user_name,
+ avatar: context.container.avatar_url,
+ avatar_shape: 'circle'
+ }
+ end
+
+ private
+
+ def user
+ context.container
+ end
+
+ def user_name
+ return user_display_name(user) if user.blocked? || !user.confirmed?
+
+ user.name
+ end
+
+ def add_menus
+ add_menu(Sidebars::UserProfile::Menus::OverviewMenu.new(context))
+ add_menu(Sidebars::UserProfile::Menus::ActivityMenu.new(context))
+ add_menu(Sidebars::UserProfile::Menus::GroupsMenu.new(context))
+ add_menu(Sidebars::UserProfile::Menus::ContributedProjectsMenu.new(context))
+ add_menu(Sidebars::UserProfile::Menus::PersonalProjectsMenu.new(context))
+ add_menu(Sidebars::UserProfile::Menus::StarredProjectsMenu.new(context))
+ add_menu(Sidebars::UserProfile::Menus::SnippetsMenu.new(context))
+ add_menu(Sidebars::UserProfile::Menus::FollowersMenu.new(context))
+ add_menu(Sidebars::UserProfile::Menus::FollowingMenu.new(context))
+ end
+ end
+ end
+end
diff --git a/lib/sidebars/user_settings/menus/access_tokens_menu.rb b/lib/sidebars/user_settings/menus/access_tokens_menu.rb
new file mode 100644
index 00000000000..f52be22e044
--- /dev/null
+++ b/lib/sidebars/user_settings/menus/access_tokens_menu.rb
@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module UserSettings
+ module Menus
+ class AccessTokensMenu < ::Sidebars::Menu
+ override :link
+ def link
+ profile_personal_access_tokens_path
+ end
+
+ override :title
+ def title
+ _('Access Tokens')
+ end
+
+ override :sprite_icon
+ def sprite_icon
+ 'token'
+ end
+
+ override :render?
+ def render?
+ !!context.current_user && !Gitlab::CurrentSettings.personal_access_tokens_disabled?
+ end
+
+ override :active_routes
+ def active_routes
+ { controller: :personal_access_tokens }
+ end
+
+ override :extra_container_html_options
+ def extra_container_html_options
+ { 'data-qa-selector': 'access_token_link' }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/sidebars/user_settings/menus/account_menu.rb b/lib/sidebars/user_settings/menus/account_menu.rb
new file mode 100644
index 00000000000..a26dee83da3
--- /dev/null
+++ b/lib/sidebars/user_settings/menus/account_menu.rb
@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module UserSettings
+ module Menus
+ class AccountMenu < ::Sidebars::Menu
+ include ::Sidebars::Concerns::RenderIfLoggedIn
+
+ override :link
+ def link
+ profile_account_path
+ end
+
+ override :title
+ def title
+ _('Account')
+ end
+
+ override :sprite_icon
+ def sprite_icon
+ 'account'
+ end
+
+ override :active_routes
+ def active_routes
+ { controller: [:accounts, :two_factor_auths] }
+ end
+
+ override :extra_container_html_options
+ def extra_container_html_options
+ { 'data-qa-selector': 'profile_account_link' }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/sidebars/user_settings/menus/active_sessions_menu.rb b/lib/sidebars/user_settings/menus/active_sessions_menu.rb
new file mode 100644
index 00000000000..f806c04e77c
--- /dev/null
+++ b/lib/sidebars/user_settings/menus/active_sessions_menu.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module UserSettings
+ module Menus
+ class ActiveSessionsMenu < ::Sidebars::Menu
+ include ::Sidebars::Concerns::RenderIfLoggedIn
+
+ override :link
+ def link
+ profile_active_sessions_path
+ end
+
+ override :title
+ def title
+ _('Active Sessions')
+ end
+
+ override :sprite_icon
+ def sprite_icon
+ 'monitor-lines'
+ end
+
+ override :active_routes
+ def active_routes
+ { controller: :active_sessions }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/sidebars/user_settings/menus/applications_menu.rb b/lib/sidebars/user_settings/menus/applications_menu.rb
new file mode 100644
index 00000000000..c71f9a9660b
--- /dev/null
+++ b/lib/sidebars/user_settings/menus/applications_menu.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module UserSettings
+ module Menus
+ class ApplicationsMenu < ::Sidebars::Menu
+ include ::Sidebars::Concerns::RenderIfLoggedIn
+
+ override :link
+ def link
+ applications_profile_path
+ end
+
+ override :title
+ def title
+ _('Applications')
+ end
+
+ override :sprite_icon
+ def sprite_icon
+ 'applications'
+ end
+
+ override :active_routes
+ def active_routes
+ { controller: 'oauth/applications' }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/sidebars/user_settings/menus/authentication_log_menu.rb b/lib/sidebars/user_settings/menus/authentication_log_menu.rb
new file mode 100644
index 00000000000..c5a27acf1fd
--- /dev/null
+++ b/lib/sidebars/user_settings/menus/authentication_log_menu.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module UserSettings
+ module Menus
+ class AuthenticationLogMenu < ::Sidebars::Menu
+ include ::Sidebars::Concerns::RenderIfLoggedIn
+
+ override :link
+ def link
+ audit_log_profile_path
+ end
+
+ override :title
+ def title
+ _('Authentication Log')
+ end
+
+ override :sprite_icon
+ def sprite_icon
+ 'log'
+ end
+
+ override :active_routes
+ def active_routes
+ { path: 'profiles#audit_log' }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/sidebars/user_settings/menus/chat_menu.rb b/lib/sidebars/user_settings/menus/chat_menu.rb
new file mode 100644
index 00000000000..54795c29b3f
--- /dev/null
+++ b/lib/sidebars/user_settings/menus/chat_menu.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module UserSettings
+ module Menus
+ class ChatMenu < ::Sidebars::Menu
+ include ::Sidebars::Concerns::RenderIfLoggedIn
+
+ override :link
+ def link
+ profile_chat_names_path
+ end
+
+ override :title
+ def title
+ _('Chat')
+ end
+
+ override :sprite_icon
+ def sprite_icon
+ 'comment'
+ end
+
+ override :active_routes
+ def active_routes
+ { controller: :chat_names }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/sidebars/user_settings/menus/comment_templates_menu.rb b/lib/sidebars/user_settings/menus/comment_templates_menu.rb
new file mode 100644
index 00000000000..da37c42bbd4
--- /dev/null
+++ b/lib/sidebars/user_settings/menus/comment_templates_menu.rb
@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module UserSettings
+ module Menus
+ class CommentTemplatesMenu < ::Sidebars::Menu
+ include UsersHelper
+
+ override :link
+ def link
+ profile_comment_templates_path
+ end
+
+ override :title
+ def title
+ _('Comment Templates')
+ end
+
+ override :sprite_icon
+ def sprite_icon
+ 'comment-lines'
+ end
+
+ override :render?
+ def render?
+ !!context.current_user && saved_replies_enabled?
+ end
+
+ override :active_routes
+ def active_routes
+ { controller: :comment_templates }
+ end
+
+ private
+
+ def current_user
+ context.current_user
+ end
+ end
+ end
+ end
+end
diff --git a/lib/sidebars/user_settings/menus/emails_menu.rb b/lib/sidebars/user_settings/menus/emails_menu.rb
new file mode 100644
index 00000000000..3b6b4ae1daf
--- /dev/null
+++ b/lib/sidebars/user_settings/menus/emails_menu.rb
@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module UserSettings
+ module Menus
+ class EmailsMenu < ::Sidebars::Menu
+ include ::Sidebars::Concerns::RenderIfLoggedIn
+
+ override :link
+ def link
+ profile_emails_path
+ end
+
+ override :title
+ def title
+ _('Emails')
+ end
+
+ override :sprite_icon
+ def sprite_icon
+ 'mail'
+ end
+
+ override :active_routes
+ def active_routes
+ { controller: :emails }
+ end
+
+ override :extra_container_html_options
+ def extra_container_html_options
+ { 'data-qa-selector': 'profile_emails_link' }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/sidebars/user_settings/menus/gpg_keys_menu.rb b/lib/sidebars/user_settings/menus/gpg_keys_menu.rb
new file mode 100644
index 00000000000..89e447aa277
--- /dev/null
+++ b/lib/sidebars/user_settings/menus/gpg_keys_menu.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module UserSettings
+ module Menus
+ class GpgKeysMenu < ::Sidebars::Menu
+ include ::Sidebars::Concerns::RenderIfLoggedIn
+
+ override :link
+ def link
+ profile_gpg_keys_path
+ end
+
+ override :title
+ def title
+ _('GPG Keys')
+ end
+
+ override :sprite_icon
+ def sprite_icon
+ 'key'
+ end
+
+ override :active_routes
+ def active_routes
+ { controller: :gpg_keys }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/sidebars/user_settings/menus/notifications_menu.rb b/lib/sidebars/user_settings/menus/notifications_menu.rb
new file mode 100644
index 00000000000..f7ea0de0cc9
--- /dev/null
+++ b/lib/sidebars/user_settings/menus/notifications_menu.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module UserSettings
+ module Menus
+ class NotificationsMenu < ::Sidebars::Menu
+ include ::Sidebars::Concerns::RenderIfLoggedIn
+
+ override :link
+ def link
+ profile_notifications_path
+ end
+
+ override :title
+ def title
+ _('Notifications')
+ end
+
+ override :sprite_icon
+ def sprite_icon
+ 'notifications'
+ end
+
+ override :active_routes
+ def active_routes
+ { controller: :notifications }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/sidebars/user_settings/menus/password_menu.rb b/lib/sidebars/user_settings/menus/password_menu.rb
new file mode 100644
index 00000000000..9a53e0c727e
--- /dev/null
+++ b/lib/sidebars/user_settings/menus/password_menu.rb
@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module UserSettings
+ module Menus
+ class PasswordMenu < ::Sidebars::Menu
+ override :link
+ def link
+ edit_profile_password_path
+ end
+
+ override :title
+ def title
+ _('Password')
+ end
+
+ override :sprite_icon
+ def sprite_icon
+ 'lock'
+ end
+
+ override :render?
+ def render?
+ !!context.current_user&.allow_password_authentication?
+ end
+
+ override :active_routes
+ def active_routes
+ { controller: :passwords }
+ end
+
+ override :extra_container_html_options
+ def extra_container_html_options
+ { 'data-qa-selector': 'profile_password_link' }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/sidebars/user_settings/menus/preferences_menu.rb b/lib/sidebars/user_settings/menus/preferences_menu.rb
new file mode 100644
index 00000000000..b6b94ec1ad9
--- /dev/null
+++ b/lib/sidebars/user_settings/menus/preferences_menu.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module UserSettings
+ module Menus
+ class PreferencesMenu < ::Sidebars::Menu
+ include ::Sidebars::Concerns::RenderIfLoggedIn
+
+ override :link
+ def link
+ profile_preferences_path
+ end
+
+ override :title
+ def title
+ _('Preferences')
+ end
+
+ override :sprite_icon
+ def sprite_icon
+ 'preferences'
+ end
+
+ override :active_routes
+ def active_routes
+ { controller: :preferences }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/sidebars/user_settings/menus/profile_menu.rb b/lib/sidebars/user_settings/menus/profile_menu.rb
new file mode 100644
index 00000000000..73119070586
--- /dev/null
+++ b/lib/sidebars/user_settings/menus/profile_menu.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module UserSettings
+ module Menus
+ class ProfileMenu < ::Sidebars::Menu
+ include ::Sidebars::Concerns::RenderIfLoggedIn
+
+ override :link
+ def link
+ profile_path
+ end
+
+ override :title
+ def title
+ _('Profile')
+ end
+
+ override :sprite_icon
+ def sprite_icon
+ 'profile'
+ end
+
+ override :active_routes
+ def active_routes
+ { path: 'profiles#show' }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/sidebars/user_settings/menus/ssh_keys_menu.rb b/lib/sidebars/user_settings/menus/ssh_keys_menu.rb
new file mode 100644
index 00000000000..8d92db0147a
--- /dev/null
+++ b/lib/sidebars/user_settings/menus/ssh_keys_menu.rb
@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module UserSettings
+ module Menus
+ class SshKeysMenu < ::Sidebars::Menu
+ include ::Sidebars::Concerns::RenderIfLoggedIn
+
+ override :link
+ def link
+ profile_keys_path
+ end
+
+ override :title
+ def title
+ _('SSH Keys')
+ end
+
+ override :sprite_icon
+ def sprite_icon
+ 'key'
+ end
+
+ override :active_routes
+ def active_routes
+ { controller: :keys }
+ end
+
+ override :extra_container_html_options
+ def extra_container_html_options
+ { 'data-qa-selector': 'ssh_keys_link' }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/sidebars/user_settings/panel.rb b/lib/sidebars/user_settings/panel.rb
new file mode 100644
index 00000000000..683a0e34570
--- /dev/null
+++ b/lib/sidebars/user_settings/panel.rb
@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Sidebars
+ module UserSettings
+ class Panel < ::Sidebars::Panel
+ override :configure_menus
+ def configure_menus
+ add_menus
+ end
+
+ override :aria_label
+ def aria_label
+ _('User settings')
+ end
+
+ override :render_raw_scope_menu_partial
+ def render_raw_scope_menu_partial
+ "shared/nav/user_settings_scope_header"
+ end
+
+ override :super_sidebar_context_header
+ def super_sidebar_context_header
+ @super_sidebar_context_header ||= {
+ title: aria_label,
+ avatar: context.current_user.avatar_url
+ }
+ end
+
+ private
+
+ def add_menus
+ add_menu(Sidebars::UserSettings::Menus::ProfileMenu.new(context))
+ add_menu(Sidebars::UserSettings::Menus::AccountMenu.new(context))
+ add_menu(Sidebars::UserSettings::Menus::ApplicationsMenu.new(context))
+ add_menu(Sidebars::UserSettings::Menus::ChatMenu.new(context))
+ add_menu(Sidebars::UserSettings::Menus::AccessTokensMenu.new(context))
+ add_menu(Sidebars::UserSettings::Menus::EmailsMenu.new(context))
+ add_menu(Sidebars::UserSettings::Menus::PasswordMenu.new(context))
+ add_menu(Sidebars::UserSettings::Menus::NotificationsMenu.new(context))
+ add_menu(Sidebars::UserSettings::Menus::SshKeysMenu.new(context))
+ add_menu(Sidebars::UserSettings::Menus::GpgKeysMenu.new(context))
+ add_menu(Sidebars::UserSettings::Menus::PreferencesMenu.new(context))
+ add_menu(Sidebars::UserSettings::Menus::CommentTemplatesMenu.new(context))
+ add_menu(Sidebars::UserSettings::Menus::ActiveSessionsMenu.new(context))
+ add_menu(Sidebars::UserSettings::Menus::AuthenticationLogMenu.new(context))
+ end
+ end
+ end
+end
+
+Sidebars::UserSettings::Panel.prepend_mod_with('Sidebars::UserSettings::Panel')
diff --git a/lib/sidebars/your_work/panel.rb b/lib/sidebars/your_work/panel.rb
index 215a2a2da09..5f917976872 100644
--- a/lib/sidebars/your_work/panel.rb
+++ b/lib/sidebars/your_work/panel.rb
@@ -18,6 +18,14 @@ module Sidebars
"shared/nav/your_work_scope_header"
end
+ override :super_sidebar_context_header
+ def super_sidebar_context_header
+ @super_sidebar_context_header ||= {
+ title: aria_label,
+ icon: 'work'
+ }
+ end
+
private
def add_menus
@@ -33,3 +41,4 @@ module Sidebars
end
end
end
+Sidebars::YourWork::Panel.prepend_mod_with('Sidebars::YourWork::Panel')
diff --git a/lib/slack/api.rb b/lib/slack/api.rb
new file mode 100644
index 00000000000..0775e783337
--- /dev/null
+++ b/lib/slack/api.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+# Client for interacting with the Slack API.
+# See https://api.slack.com/web.
+module Slack
+ class API
+ BASE_URL = 'https://slack.com/api'
+ BASE_HEADERS = { 'Content-Type' => 'application/json; charset=utf-8' }.freeze
+
+ def initialize(slack_installation)
+ @token = slack_installation.bot_access_token
+
+ raise ArgumentError, "No token for slack installation #{slack_installation.id}" unless @token
+ end
+
+ def post(api_method, payload)
+ url = "#{BASE_URL}/#{api_method}"
+ headers = BASE_HEADERS.merge('Authorization' => "Bearer #{token}")
+
+ Gitlab::HTTP.post(url, body: payload.to_json, headers: headers)
+ end
+
+ private
+
+ attr_reader :token
+ end
+end
diff --git a/lib/slack/block_kit/app_home_opened.rb b/lib/slack/block_kit/app_home_opened.rb
new file mode 100644
index 00000000000..f67cdca85d9
--- /dev/null
+++ b/lib/slack/block_kit/app_home_opened.rb
@@ -0,0 +1,173 @@
+# frozen_string_literal: true
+
+# Builds the BlockKit UI JSON payload to respond to the Slack `app_home_opened` event.
+#
+# See:
+# - https://api.slack.com/block-kit/building
+# - https://api.slack.com/events/app_home_opened
+module Slack
+ module BlockKit
+ class AppHomeOpened
+ include ActionView::Helpers::AssetUrlHelper
+ include Gitlab::Routing.url_helpers
+
+ def initialize(slack_user_id, slack_workspace_id, slack_gitlab_user_connection, slack_installation)
+ @slack_user_id = slack_user_id
+ @slack_workspace_id = slack_workspace_id
+ @slack_gitlab_user_connection = slack_gitlab_user_connection
+ @slack_installation = slack_installation
+ end
+
+ def build
+ {
+ type: "home",
+ blocks: [
+ header,
+ section_introduction,
+ section_notifications_heading,
+ section_notifications,
+ section_slash_commands_heading,
+ section_slash_commands,
+ section_slash_commands_connect,
+ section_connect_gitlab_account
+ ]
+ }
+ end
+
+ private
+
+ attr_reader :slack_user_id, :slack_workspace_id, :slack_gitlab_user_connection, :slack_installation
+
+ def header
+ {
+ type: "header",
+ text: {
+ type: "plain_text",
+ text: format(
+ s_("Slack|%{emoji}Welcome to GitLab for Slack!"),
+ emoji: '✨ '
+ ),
+ emoji: true
+ }
+ }
+ end
+
+ def section_introduction
+ section(
+ format(
+ s_("Slack|GitLab for Slack now supports channel-based notifications. " \
+ "Let your team know when new issues are created or new CI/CD jobs are run." \
+ "%{startMarkup}Learn more%{endMarkup}."),
+ startMarkup: " <#{help_page_url('integration/slash_commands')}|",
+ endMarkup: ">"
+ )
+ )
+ end
+
+ def section_notifications_heading
+ section(
+ format(
+ s_("Slack|%{asterisk}Channel notifications%{asterisk}"),
+ asterisk: '*'
+ )
+ )
+ end
+
+ def section_notifications
+ section(
+ format(
+ s_("Slack|To start using notifications, " \
+ "%{startMarkup}enable the GitLab for Slack app integration%{endMarkup} in your project settings."),
+ startMarkup: "<#{help_page_url('user/project/integrations/gitlab_slack_application',
+ anchor: 'configuration')}|",
+ endMarkup: ">"
+ )
+ )
+ end
+
+ def section_slash_commands_heading
+ section(
+ format(
+ s_("Slack|%{asterisk}Slash commands%{asterisk}"),
+ asterisk: '*'
+ )
+ )
+ end
+
+ def section_slash_commands
+ section(
+ format(
+ s_("Slack|Control GitLab from Slack with " \
+ "%{startMarkup}slash commands%{endMarkup}. For a list of available commands, enter %{command}."),
+ startMarkup: "<#{help_page_url('user/project/integrations/gitlab_slack_application', anchor: 'usage')}|",
+ endMarkup: ">",
+ command: "`/gitlab help`"
+ )
+ )
+ end
+
+ def section_slash_commands_connect
+ section(
+ s_("Slack|To start using slash commands, connect your GitLab account.")
+ )
+ end
+
+ def section_connect_gitlab_account
+ if slack_gitlab_user_connection.present?
+ section_gitlab_account_connected
+ else
+ actions_gitlab_account_not_connected
+ end
+ end
+
+ def section_gitlab_account_connected
+ user = slack_gitlab_user_connection.user
+
+ section(
+ format(
+ s_("Slack|%{emoji}Connected to GitLab account %{account}."),
+ emoji: '✅ ',
+ account: "<#{Gitlab::UrlBuilder.build(user)}|#{user.to_reference}>"
+ )
+ )
+ end
+
+ def actions_gitlab_account_not_connected
+ account_connection_url = ChatNames::AuthorizeUserService.new(
+ {
+ team_id: slack_workspace_id,
+ user_id: slack_user_id,
+ team_domain: slack_workspace_id,
+ user_name: 'Slack'
+ }
+ ).execute
+
+ {
+ type: "actions",
+ elements: [
+ {
+ type: "button",
+ text: {
+ type: "plain_text",
+ text: s_("Slack|Connect your GitLab account"),
+ emoji: true
+ },
+ style: "primary",
+ url: account_connection_url
+ }
+ ]
+ }
+ end
+
+ def section(text)
+ {
+ type: "section",
+ text: {
+ type: "mrkdwn",
+ text: text
+ }
+ }
+ end
+ end
+ end
+end
diff --git a/lib/slack/block_kit/incident_management/incident_modal_opened.rb b/lib/slack/block_kit/incident_management/incident_modal_opened.rb
new file mode 100644
index 00000000000..e34285017b0
--- /dev/null
+++ b/lib/slack/block_kit/incident_management/incident_modal_opened.rb
@@ -0,0 +1,343 @@
+# frozen_string_literal: true
+
+module Slack
+ module BlockKit
+ module IncidentManagement
+ class IncidentModalOpened
+ # See https://api.slack.com/reference/block-kit/composition-objects#option for the text limit.
+ # Dropdown menu items can have max length of 75 chars.
+ MAX_CHAR_LENGTH = 75
+
+ def initialize(projects, response_url)
+ @projects = projects
+ @response_url = response_url
+ end
+
+ def build
+ {
+ type: "modal",
+ title: modal_title,
+ submit: submit_button,
+ close: close_button,
+ notify_on_close: true,
+ callback_id: 'incident_modal',
+ private_metadata: response_url,
+ blocks: [
+ title_block,
+ details_selection_block,
+ status_and_assignee_block,
+ label_block,
+ confidential_block,
+ incident_description_block,
+ zoom_link_block
+ ]
+ }
+ end
+
+ private
+
+ attr_reader :projects, :response_url
+
+ def modal_title
+ {
+ type: "plain_text",
+ text: _("New incident")
+ }
+ end
+
+ def submit_button
+ {
+ type: "plain_text",
+ text: _("Create")
+ }
+ end
+
+ def close_button
+ {
+ type: "plain_text",
+ text: _("Cancel")
+ }
+ end
+
+ def formatted_project_path(path)
+ path.truncate(MAX_CHAR_LENGTH)
+ end
+
+ def title_block
+ {
+ type: "input",
+ block_id: "title_input",
+ label: {
+ type: "plain_text",
+ text: _("Title")
+ },
+ element: {
+ type: "plain_text_input",
+ action_id: "title",
+ placeholder: {
+ type: "plain_text",
+ text: _("Incident title")
+ },
+ focus_on_load: true
+ }
+ }
+ end
+
+ def project_selection
+ {
+ type: "static_select",
+ action_id: "incident_management_project",
+ placeholder: {
+ type: "plain_text",
+ text: _("Select project")
+ },
+ confirm: confirmation_dialog(
+ s_("SlackModal|Are you sure you want to change the project?"),
+ [
+ s_("SlackModal|If you change the project, you'll lose any text entered in the form."),
+ s_("SlackModal|If you've entered some text, consider saving it somewhere to avoid losing any content.")
+ ].join("\n")
+ ),
+ options: construct_project_selector,
+ initial_option: project_selector_option(projects.first)
+ }
+ end
+
+ def status_selection
+ {
+ type: "static_select",
+ action_id: "status",
+ placeholder: {
+ type: "plain_text",
+ text: s_("Status (optional)")
+ },
+ options: [
+ {
+ text: {
+ type: "plain_text",
+ text: s_("EscalationStatus|Triggered")
+ },
+ value: "triggered"
+ },
+ {
+ text: {
+ type: "plain_text",
+ text: s_("EscalationStatus|Acknowledged")
+ },
+ value: "acknowledged"
+ },
+ {
+ text: {
+ type: "plain_text",
+ text: s_("EscalationStatus|Resolved")
+ },
+ value: "resolved"
+ }
+ ]
+ }
+ end
+
+ def severity_selection
+ {
+ type: "static_select",
+ action_id: "severity",
+ placeholder: {
+ type: "plain_text",
+ text: s_("Select severity (optional)")
+ },
+ options: [
+ {
+ text: {
+ type: "plain_text",
+ text: s_("Critical - S1")
+ },
+ value: "critical"
+ },
+ {
+ text: {
+ type: "plain_text",
+ text: s_("High - S2")
+ },
+ value: "high"
+ },
+ {
+ text: {
+ type: "plain_text",
+ text: s_("Medium - S3")
+ },
+ value: "medium"
+ },
+ {
+ text: {
+ type: "plain_text",
+ text: s_("Low - S4")
+ },
+ value: "low"
+ }
+ ]
+ }
+ end
+
+ def assignee_selection
+ {
+ type: "external_select",
+ action_id: "assignee",
+ placeholder: {
+ type: "plain_text",
+ text: s_("Assignee (optional)")
+ }
+ }
+ end
+
+ def label_block
+ {
+ type: "input",
+ optional: true,
+ block_id: "label_selector",
+ label: {
+ type: "plain_text",
+ text: s_("Labels")
+ },
+ element: {
+ type: "multi_external_select",
+ action_id: "labels",
+ placeholder: {
+ type: "plain_text",
+ text: s_("Select labels (optional)")
+ }
+ }
+ }
+ end
+
+ def confidential_block
+ {
+ type: "actions",
+ block_id: "confidentiality",
+ elements: [
+ {
+ type: "checkboxes",
+ action_id: "confidential",
+ options: [
+ {
+ value: "confidential",
+ text: {
+ type: "plain_text",
+ text: _("Confidential")
+ }
+ }
+ ]
+ }
+ ]
+ }
+ end
+
+ def details_selection_block
+ {
+ type: "actions",
+ block_id: "project_and_severity_selector",
+ elements: [
+ project_selection,
+ severity_selection
+ ]
+ }
+ end
+
+ def status_and_assignee_block
+ {
+ type: "actions",
+ block_id: "status_and_assignee_selector",
+ elements: [
+ status_selection,
+ assignee_selection
+ ]
+ }
+ end
+
+ def incident_description_block
+ {
+ block_id: "incident_description",
+ type: "input",
+ element: {
+ type: "plain_text_input",
+ multiline: true,
+ action_id: "description",
+ placeholder: {
+ type: "plain_text",
+ text: [
+ _("Write a description..."),
+ _("[Supports GitLab-flavored markdown, including quick actions]")
+ ].join("\n\n")
+ },
+ initial_value: project_incident_template(projects.first)
+ },
+ label: {
+ type: "plain_text",
+ text: _("Description")
+ }
+ }
+ end
+
+ def zoom_link_block
+ {
+ type: "input",
+ block_id: "zoom",
+ optional: true,
+ element: {
+ type: "plain_text_input",
+ action_id: "link",
+ placeholder: {
+ type: "plain_text",
+ text: format(_("%{url} (optional)"), url: 'https://example.zoom.us')
+ }
+ },
+ label: {
+ type: "plain_text",
+ text: "Zoom"
+ }
+ }
+ end
+
+ def construct_project_selector
+ projects.map do |project|
+ project_selector_option(project)
+ end
+ end
+
+ def project_selector_option(project)
+ {
+ text: {
+ type: "plain_text",
+ text: formatted_project_path(project.full_path)
+ },
+ value: project.id.to_s
+ }
+ end
+
+ def project_incident_template(project)
+ project.incident_management_setting&.issue_template_content.to_s
+ end
+
+ def confirmation_dialog(question, warning)
+ {
+ title: {
+ type: "plain_text",
+ text: question
+ },
+ text: {
+ type: "plain_text",
+ text: warning
+ },
+ confirm: {
+ type: "plain_text",
+ text: "Yes"
+ },
+ deny: {
+ type: "plain_text",
+ text: "No"
+ }
+ }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/support/nginx/gitlab b/lib/support/nginx/gitlab
index 74eb8634d58..19a88296120 100644
--- a/lib/support/nginx/gitlab
+++ b/lib/support/nginx/gitlab
@@ -28,7 +28,7 @@ map $http_upgrade $connection_upgrade_gitlab {
}
## NGINX 'combined' log format with filtered query strings
-log_format gitlab_access $remote_addr - $remote_user [$time_local] "$request_method $gitlab_filtered_request_uri $server_protocol" $status $body_bytes_sent "$gitlab_filtered_http_referer" "$http_user_agent";
+log_format gitlab_access '$remote_addr - $remote_user [$time_local] "$request_method $gitlab_filtered_request_uri $server_protocol" $status $body_bytes_sent "$gitlab_filtered_http_referer" "$http_user_agent"';
## Remove private_token from the request URI
# In: /foo?private_token=unfiltered&authenticity_token=unfiltered&feed_token=unfiltered&...
diff --git a/lib/support/nginx/gitlab-pages-ssl b/lib/support/nginx/gitlab-pages-ssl
index 900d91e0575..a883d151a96 100644
--- a/lib/support/nginx/gitlab-pages-ssl
+++ b/lib/support/nginx/gitlab-pages-ssl
@@ -38,8 +38,10 @@ server {
ssl_session_cache shared:SSL:10m;
ssl_session_tickets off;
- ssl_protocols TLSv1.2 TLSv1.3;
- ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384;
+ # These settings are in line with the modern settings from https://ssl-config.mozilla.org/
+ # and are supported by all still-supported browsers since 2019. If you have specific needs
+ # for older settings, please consult the intermediate settings there.
+ ssl_protocols TLSv1.3;
ssl_prefer_server_ciphers off;
## See app/controllers/application_controller.rb for headers set
@@ -53,11 +55,6 @@ server {
# ssl_stapling_verify on;
# ssl_trusted_certificate /etc/nginx/ssl/stapling.trusted.crt;
- ## [Optional] Generate a stronger DHE parameter:
- ## sudo openssl dhparam -out /etc/ssl/certs/dhparam.pem 4096
- ##
- # ssl_dhparam /etc/ssl/certs/dhparam.pem;
-
## [Optional] Enable HTTP Strict Transport Security
# add_header Strict-Transport-Security "max-age=63072000; includeSubDomains";
diff --git a/lib/support/nginx/gitlab-ssl b/lib/support/nginx/gitlab-ssl
index 435b9055929..02443703423 100644
--- a/lib/support/nginx/gitlab-ssl
+++ b/lib/support/nginx/gitlab-ssl
@@ -33,7 +33,7 @@ map $http_upgrade $connection_upgrade_gitlab_ssl {
## NGINX 'combined' log format with filtered query strings
-log_format gitlab_ssl_access $remote_addr - $remote_user [$time_local] "$request_method $gitlab_ssl_filtered_request_uri $server_protocol" $status $body_bytes_sent "$gitlab_ssl_filtered_http_referer" "$http_user_agent";
+log_format gitlab_ssl_access '$remote_addr - $remote_user [$time_local] "$request_method $gitlab_ssl_filtered_request_uri $server_protocol" $status $body_bytes_sent "$gitlab_ssl_filtered_http_referer" "$http_user_agent"';
## Remove private_token from the request URI
# In: /foo?private_token=unfiltered&authenticity_token=unfiltered&feed_token=unfiltered&...
@@ -97,8 +97,10 @@ server {
ssl_session_cache shared:SSL:10m;
ssl_session_tickets off;
- ssl_protocols TLSv1.2 TLSv1.3;
- ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384;
+ # These settings are in line with the modern settings from https://ssl-config.mozilla.org/
+ # and are supported by all still-supported browsers since 2019. If you have specific needs
+ # for older settings, please consult the intermediate settings there.
+ ssl_protocols TLSv1.3;
ssl_prefer_server_ciphers off;
## See app/controllers/application_controller.rb for headers set
@@ -114,11 +116,6 @@ server {
# resolver 208.67.222.222 208.67.222.220 valid=300s; # Can change to your DNS resolver if desired
# resolver_timeout 5s;
- ## [Optional] Generate a stronger DHE parameter:
- ## sudo openssl dhparam -out /etc/ssl/certs/dhparam.pem 4096
- ##
- # ssl_dhparam /etc/ssl/certs/dhparam.pem;
-
## [Optional] Enable HTTP Strict Transport Security
# add_header Strict-Transport-Security "max-age=63072000; includeSubDomains";
diff --git a/lib/support/nginx/registry-ssl b/lib/support/nginx/registry-ssl
index be16037629b..50ecb855e24 100644
--- a/lib/support/nginx/registry-ssl
+++ b/lib/support/nginx/registry-ssl
@@ -34,15 +34,12 @@ server {
ssl_session_cache shared:SSL:10m;
ssl_session_tickets off;
- ssl_protocols TLSv1.2 TLSv1.3;
- ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384;
+ # These settings are in line with the modern settings from https://ssl-config.mozilla.org/
+ # and are supported by all still-supported browsers since 2019. If you have specific needs
+ # for older settings, please consult the intermediate settings there.
+ ssl_protocols TLSv1.3;
ssl_prefer_server_ciphers off;
- ## [Optional] Generate a stronger DHE parameter:
- ## sudo openssl dhparam -out /etc/ssl/certs/dhparam.pem 4096
- ##
- # ssl_dhparam /etc/ssl/certs/dhparam.pem;
-
## [Optional] Enable HTTP Strict Transport Security
# add_header Strict-Transport-Security "max-age=63072000; includeSubDomains";
diff --git a/lib/system_check/app/redis_version_check.rb b/lib/system_check/app/redis_version_check.rb
index a6ff2405390..d43c7f4a497 100644
--- a/lib/system_check/app/redis_version_check.rb
+++ b/lib/system_check/app/redis_version_check.rb
@@ -5,9 +5,7 @@ require 'redis'
module SystemCheck
module App
class RedisVersionCheck < SystemCheck::BaseCheck
- # Redis 5.x will be deprecated
- # https://gitlab.com/gitlab-org/gitlab/-/issues/331468
- MIN_REDIS_VERSION = '5.0.0'
+ MIN_REDIS_VERSION = '6.0.0'
RECOMMENDED_REDIS_VERSION = "6.0.0"
set_name "Redis version >= #{RECOMMENDED_REDIS_VERSION}?"
diff --git a/lib/tasks/benchmark.rake b/lib/tasks/benchmark.rake
index 6deafb2c351..cb95e53a0ee 100644
--- a/lib/tasks/benchmark.rake
+++ b/lib/tasks/benchmark.rake
@@ -3,9 +3,15 @@
return if Rails.env.production?
namespace :benchmark do
- desc 'Benchmark | Banzai pipeline/filters'
+ desc 'Benchmark | Banzai pipeline/filters (optionally specify FILTER=xxxxxFilter)'
RSpec::Core::RakeTask.new(:banzai) do |t|
t.pattern = 'spec/benchmarks/banzai_benchmark.rb'
+ t.rspec_opts = if ENV.key?('FILTER')
+ ['--tag specific_filter']
+ else
+ ['--tag \~specific_filter']
+ end
+
ENV['BENCHMARK'] = '1'
end
end
diff --git a/lib/tasks/contracts/merge_requests.rake b/lib/tasks/contracts/merge_requests.rake
index 5a6186d393d..049e4e41092 100644
--- a/lib/tasks/contracts/merge_requests.rake
+++ b/lib/tasks/contracts/merge_requests.rake
@@ -38,7 +38,7 @@ namespace :contracts do
end
desc 'Run all merge request contract tests'
- task 'test:merge_requests', :contract_merge_requests do |_t, arg|
+ task 'test:merge_requests', :contract_merge_requests do |_t|
errors = %w[get_diffs_batch get_diffs_metadata get_discussions].each_with_object([]) do |task, err|
Rake::Task["contracts:merge_requests:pact:verify:#{task}"].execute
rescue StandardError, SystemExit
diff --git a/lib/tasks/contracts/pipeline_schedules.rake b/lib/tasks/contracts/pipeline_schedules.rake
index f3e65b94940..2733ad41de6 100644
--- a/lib/tasks/contracts/pipeline_schedules.rake
+++ b/lib/tasks/contracts/pipeline_schedules.rake
@@ -20,7 +20,7 @@ namespace :contracts do
end
desc 'Run all pipeline schedule contract tests'
- task 'test:pipeline_schedules', :contract_pipeline_schedules do |_t, arg|
+ task 'test:pipeline_schedules', :contract_pipeline_schedules do |_t|
errors = %w[
update_pipeline_schedule
].each_with_object([]) do |task, err|
diff --git a/lib/tasks/contracts/pipelines.rake b/lib/tasks/contracts/pipelines.rake
index 13c973f1358..08e0a8b0319 100644
--- a/lib/tasks/contracts/pipelines.rake
+++ b/lib/tasks/contracts/pipelines.rake
@@ -47,7 +47,7 @@ namespace :contracts do
end
desc 'Run all pipeline contract tests'
- task 'test:pipelines', :contract_pipelines do |_t, arg|
+ task 'test:pipelines', :contract_pipelines do |_t|
errors = %w[
create_a_new_pipeline
get_list_project_pipelines
diff --git a/lib/tasks/db_obsolete_ignored_columns.rake b/lib/tasks/db_obsolete_ignored_columns.rake
index a689a9bf2d8..c71e3169723 100644
--- a/lib/tasks/db_obsolete_ignored_columns.rake
+++ b/lib/tasks/db_obsolete_ignored_columns.rake
@@ -5,9 +5,9 @@ task 'db:obsolete_ignored_columns' => :environment do
list = Gitlab::Database::ObsoleteIgnoredColumns.new.execute
if list.empty?
- puts 'No obsolete `ignored_columns` found.'
+ puts 'No obsolete `ignored_columns` definitions found.'
else
- puts 'The following `ignored_columns` are obsolete and can be removed:'
+ puts 'The following `ignored_columns` definitions are obsolete and can be removed:'
list.each do |name, ignored_columns|
puts "#{name}:"
diff --git a/lib/tasks/gettext.rake b/lib/tasks/gettext.rake
index 17f9414ad52..825388461bc 100644
--- a/lib/tasks/gettext.rake
+++ b/lib/tasks/gettext.rake
@@ -7,50 +7,35 @@ namespace :gettext do
# See: https://gitlab.com/gitlab-org/gitlab-foss/issues/33014#note_31218998
FileUtils.touch(pot_file_path)
- Rake::Task['gettext:po_to_json'].invoke
+ command = [
+ "node", "./scripts/frontend/po_to_json.js",
+ "--locale-root", Rails.root.join('locale').to_s,
+ "--output-dir", Rails.root.join('app/assets/javascripts/locale').to_s
+ ]
+
+ abort 'Error: Unable to convert gettext files to js.'.color(:red) unless Kernel.system(*command)
end
desc 'Regenerate gitlab.pot file'
task :regenerate do
+ require_relative "../../tooling/lib/tooling/gettext_extractor"
ensure_locale_folder_presence!
- # Clean up folders that do not contain a gitlab.po file
- Pathname.new(locale_path).children.each do |child|
- next unless child.directory?
-
- folder_path = child.to_path
-
- if File.exist?("#{folder_path}/gitlab.po")
- # remove all translated files to speed up finding
- FileUtils.rm Dir["#{folder_path}/gitlab.*"]
- else
- # remove empty translation folders so we don't generate un-needed .po files
- puts "Deleting #{folder_path} as it does not contain a 'gitlab.po' file."
-
- FileUtils.rm_r folder_path
- end
- end
-
# remove the `pot` file to ensure it's completely regenerated
FileUtils.rm_f(pot_file_path)
- Rake::Task['gettext:find'].invoke
-
- # leave only the required changes.
- unless system(*%w(git -c core.hooksPath=/dev/null checkout -- locale/*/gitlab.po))
- raise 'failed to cleanup generated locale/*/gitlab.po files'
- end
+ extractor = Tooling::GettextExtractor.new(
+ glob_base: Rails.root
+ )
+ File.write(pot_file_path, extractor.generate_pot)
raise 'gitlab.pot file not generated' unless File.exist?(pot_file_path)
- # Remove timestamps from the pot file
- pot_content = File.read pot_file_path
- pot_content.gsub!(/^"POT?\-(?:Creation|Revision)\-Date\:.*\n/, '')
- File.write pot_file_path, pot_content
-
puts <<~MSG
- All done. Please commit the changes to `locale/gitlab.pot`.
+ All done. Please commit the changes to `locale/gitlab.pot`.
+ Tip: For even faster regeneration, directly run the following command:
+ tooling/bin/gettext_extractor locale/gitlab.pot
MSG
end
@@ -86,19 +71,7 @@ namespace :gettext do
end
end
- task :updated_check do
- # Removing all pre-translated files speeds up `gettext:find` as the
- # files don't need to be merged.
- # Having `LC_MESSAGES/gitlab.mo files present also confuses the output.
- FileUtils.rm Dir['locale/**/gitlab.*']
- FileUtils.rm_f pot_file_path
-
- # `gettext:find` writes touches to temp files to `stderr` which would cause
- # `static-analysis` to report failures. We can ignore these.
- silence_stderr do
- Rake::Task['gettext:find'].invoke
- end
-
+ task updated_check: [:regenerate] do
pot_diff = `git diff -- #{pot_file_path} | grep -E '^(\\+|-)msgid'`.strip
# reset the locale folder for potential next tasks
@@ -108,7 +81,7 @@ namespace :gettext do
raise <<~MSG
Changes in translated strings found, please update file `#{pot_file_path}` by running:
- bin/rake gettext:regenerate
+ tooling/bin/gettext_extractor locale/gitlab.pot
Then commit and push the resulting changes to `#{pot_file_path}`.
@@ -121,17 +94,6 @@ namespace :gettext do
private
- # Customize list of translatable files
- # See: https://github.com/grosser/gettext_i18n_rails#customizing-list-of-translatable-files
- def files_to_translate
- folders = %W(ee app lib config #{locale_path}).join(',')
- exts = %w(rb erb haml slim rhtml js jsx vue handlebars hbs mustache).join(',')
-
- Dir.glob(
- "{#{folders}}/**/*.{#{exts}}"
- )
- end
-
def report_errors_for_file(file, errors_for_file)
puts "Errors in `#{file}`:"
@@ -159,7 +121,7 @@ namespace :gettext do
def ensure_locale_folder_presence!
unless Dir.exist?(locale_path)
raise <<~MSG
- Cannot find '#{locale_path}' folder. Please ensure you're running this task from the gitlab repo.
+ Cannot find '#{locale_path}' folder. Please ensure you're running this task from the gitlab repo.
MSG
end
diff --git a/lib/tasks/gitlab/assets.rake b/lib/tasks/gitlab/assets.rake
index d8c0b1007e6..2522488f579 100644
--- a/lib/tasks/gitlab/assets.rake
+++ b/lib/tasks/gitlab/assets.rake
@@ -1,7 +1,5 @@
# frozen_string_literal: true
-require 'fileutils'
-
module Tasks
module Gitlab
module Assets
@@ -18,6 +16,13 @@ module Tasks
babel.config.js
config/webpack.config.js
].freeze
+ # Ruby gems might emit assets which have an impact on compilation
+ # or have a direct impact on asset compilation (e.g. scss) and therefore
+ # we should compile when these change
+ RAILS_ASSET_FILES = %w[
+ Gemfile
+ Gemfile.lock
+ ].freeze
EXCLUDE_PATTERNS = %w[
app/assets/javascripts/locale/**/app.js
].freeze
@@ -50,6 +55,9 @@ module Tasks
Digest::SHA256.hexdigest(assets_sha256).tap { |sha256| puts "=> SHA256 generated in #{Time.now - start_time}: #{sha256}" if verbose }
end
+ # Files listed here should match the list in:
+ # .assets-compilation-patterns in .gitlab/ci/rules.gitlab-ci.yml
+ # So we make sure that any impacting changes we do rebuild cache
def self.assets_impacting_compilation
assets_folders = FOSS_ASSET_FOLDERS
assets_folders += EE_ASSET_FOLDERS if ::Gitlab.ee?
@@ -57,6 +65,7 @@ module Tasks
asset_files = Dir.glob(JS_ASSET_PATTERNS)
asset_files += JS_ASSET_FILES
+ asset_files += RAILS_ASSET_FILES
assets_folders.each do |folder|
asset_files.concat(Dir.glob(["#{folder}/**/*.*"]))
@@ -78,6 +87,8 @@ namespace :gitlab do
desc 'GitLab | Assets | Compile all frontend assets'
task :compile do
+ require 'fileutils'
+
require_dependency 'gitlab/task_helpers'
puts "Assets SHA256 for `master`: #{Tasks::Gitlab::Assets.master_assets_sha256.inspect}"
@@ -86,9 +97,9 @@ namespace :gitlab do
if Tasks::Gitlab::Assets.head_assets_sha256 != Tasks::Gitlab::Assets.master_assets_sha256
FileUtils.rm_rf([Tasks::Gitlab::Assets::PUBLIC_ASSETS_DIR] + Dir.glob('app/assets/javascripts/locale/**/app.js'))
- # gettext:po_to_json needs to run before rake:assets:precompile because
+ # gettext:compile needs to run before rake:assets:precompile because
# app/assets/javascripts/locale/**/app.js are pre-compiled by Sprockets
- Gitlab::TaskHelpers.invoke_and_time_task('gettext:po_to_json')
+ Gitlab::TaskHelpers.invoke_and_time_task('gettext:compile')
Gitlab::TaskHelpers.invoke_and_time_task('rake:assets:precompile')
log_path = ENV['WEBPACK_COMPILE_LOG_PATH']
diff --git a/lib/tasks/gitlab/background_migrations.rake b/lib/tasks/gitlab/background_migrations.rake
index e0699d5eb41..eca51c345d1 100644
--- a/lib/tasks/gitlab/background_migrations.rake
+++ b/lib/tasks/gitlab/background_migrations.rake
@@ -46,20 +46,22 @@ namespace :gitlab do
end
desc 'Display the status of batched background migrations'
- task status: :environment do |_, args|
- Gitlab::Database.database_base_models.each do |name, model|
- display_migration_status(name, model.connection)
+ task status: :environment do |_, _args|
+ Gitlab::Database.database_base_models.each do |database_name, model|
+ next unless Gitlab::Database.has_database?(database_name)
+
+ display_migration_status(database_name, model.connection)
end
end
namespace :status do
- ActiveRecord::Tasks::DatabaseTasks.for_each(databases) do |name|
- next if name.to_s == 'geo'
+ ActiveRecord::Tasks::DatabaseTasks.for_each(databases) do |database_name|
+ next if database_name.to_s == 'geo'
- desc "Gitlab | DB | Display the status of batched background migrations on #{name} database"
- task name => :environment do |_, args|
- model = Gitlab::Database.database_base_models[name]
- display_migration_status(name, model.connection)
+ desc "Gitlab | DB | Display the status of batched background migrations on #{database_name} database"
+ task database_name => :environment do |_, _args|
+ model = Gitlab::Database.database_base_models[database_name]
+ display_migration_status(database_name, model.connection)
end
end
end
diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake
index 06a032316c5..22e1d903c8d 100644
--- a/lib/tasks/gitlab/backup.rake
+++ b/lib/tasks/gitlab/backup.rake
@@ -1,209 +1,209 @@
# frozen_string_literal: true
+module Tasks
+ module Gitlab
+ module Backup
+ PID = Process.pid.freeze
+ PID_FILE = "#{Rails.application.root}/tmp/backup_restore.pid"
+
+ def self.create_backup
+ lock_backup do
+ ::Gitlab::TaskHelpers.warn_user_is_not_gitlab
+
+ ::Backup::Manager.new(backup_progress).create
+ end
+ end
+
+ def self.restore_backup
+ lock_backup do
+ ::Gitlab::TaskHelpers.warn_user_is_not_gitlab
+
+ ::Backup::Manager.new(backup_progress).restore
+ end
+ end
+
+ def self.create_task(task)
+ lock_backup do
+ ::Backup::Manager.new(backup_progress).run_create_task(task)
+ end
+ end
+
+ def self.restore_task(task)
+ lock_backup do
+ ::Backup::Manager.new(backup_progress).run_restore_task(task)
+ end
+ end
+
+ def self.backup_progress
+ if ENV['CRON']
+ # We need an object we can say 'puts' and 'print' to; let's use a StringIO.
+ StringIO.new
+ else
+ $stdout
+ end
+ end
+
+ def self.lock_backup
+ File.open(PID_FILE, File::RDWR | File::CREAT) do |f|
+ f.flock(File::LOCK_EX)
+
+ file_content = f.read
+
+ read_pid(file_content) unless file_content.blank?
+
+ f.rewind
+ f.write(PID)
+ f.flush
+ ensure
+ f.flock(File::LOCK_UN)
+ end
+
+ begin
+ yield
+ ensure
+ backup_progress.puts(
+ "#{Time.current} " + '-- Deleting backup and restore PID file ... '.color(:blue) + 'done'.color(:green)
+ )
+ File.delete(PID_FILE)
+ end
+ end
+
+ def self.read_pid(file_content)
+ Process.getpgid(file_content.to_i)
+
+ backup_progress.puts(<<~MESSAGE.color(:red))
+ Backup and restore in progress:
+ There is a backup and restore task in progress (PID #{file_content}).
+ Try to run the current task once the previous one ends.
+ MESSAGE
+
+ exit 1
+ rescue Errno::ESRCH
+ backup_progress.puts(<<~MESSAGE.color(:blue))
+ The PID file #{PID_FILE} exists and contains #{file_content}, but the process is not running.
+ The PID file will be rewritten with the current process ID #{PID}.
+ MESSAGE
+ end
+
+ private_class_method :backup_progress, :lock_backup, :read_pid
+ end
+ end
+end
+
namespace :gitlab do
require 'active_record/fixtures'
+ require 'stringio'
namespace :backup do
- PID = Process.pid.freeze
- PID_FILE = "#{Rails.application.root}/tmp/backup_restore.pid"
-
# Create backup of GitLab system
desc 'GitLab | Backup | Create a backup of the GitLab system'
task create: :gitlab_environment do
- lock do
- warn_user_is_not_gitlab
-
- Backup::Manager.new(progress).create
- end
+ Tasks::Gitlab::Backup.create_backup
end
# Restore backup of GitLab system
desc 'GitLab | Backup | Restore a previously created backup'
task restore: :gitlab_environment do
- lock do
- warn_user_is_not_gitlab
-
- Backup::Manager.new(progress).restore
- end
+ Tasks::Gitlab::Backup.restore_backup
end
namespace :repo do
task create: :gitlab_environment do
- lock do
- Backup::Manager.new(progress).run_create_task('repositories')
- end
+ Tasks::Gitlab::Backup.create_task('repositories')
end
task restore: :gitlab_environment do
- lock do
- Backup::Manager.new(progress).run_restore_task('repositories')
- end
+ Tasks::Gitlab::Backup.restore_task('repositories')
end
end
namespace :db do
task create: :gitlab_environment do
- lock do
- Backup::Manager.new(progress).run_create_task('db')
- end
+ Tasks::Gitlab::Backup.create_task('db')
end
task restore: :gitlab_environment do
- lock do
- Backup::Manager.new(progress).run_restore_task('db')
- end
+ Tasks::Gitlab::Backup.restore_task('db')
end
end
namespace :builds do
task create: :gitlab_environment do
- lock do
- Backup::Manager.new(progress).run_create_task('builds')
- end
+ Tasks::Gitlab::Backup.create_task('builds')
end
task restore: :gitlab_environment do
- lock do
- Backup::Manager.new(progress).run_restore_task('builds')
- end
+ Tasks::Gitlab::Backup.restore_task('builds')
end
end
namespace :uploads do
task create: :gitlab_environment do
- lock do
- Backup::Manager.new(progress).run_create_task('uploads')
- end
+ Tasks::Gitlab::Backup.create_task('uploads')
end
task restore: :gitlab_environment do
- lock do
- Backup::Manager.new(progress).run_restore_task('uploads')
- end
+ Tasks::Gitlab::Backup.restore_task('uploads')
end
end
namespace :artifacts do
task create: :gitlab_environment do
- lock do
- Backup::Manager.new(progress).run_create_task('artifacts')
- end
+ Tasks::Gitlab::Backup.create_task('artifacts')
end
task restore: :gitlab_environment do
- lock do
- Backup::Manager.new(progress).run_restore_task('artifacts')
- end
+ Tasks::Gitlab::Backup.restore_task('artifacts')
end
end
namespace :pages do
task create: :gitlab_environment do
- lock do
- Backup::Manager.new(progress).run_create_task('pages')
- end
+ Tasks::Gitlab::Backup.create_task('pages')
end
task restore: :gitlab_environment do
- lock do
- Backup::Manager.new(progress).run_restore_task('pages')
- end
+ Tasks::Gitlab::Backup.restore_task('pages')
end
end
namespace :lfs do
task create: :gitlab_environment do
- lock do
- Backup::Manager.new(progress).run_create_task('lfs')
- end
+ Tasks::Gitlab::Backup.create_task('lfs')
end
task restore: :gitlab_environment do
- lock do
- Backup::Manager.new(progress).run_restore_task('lfs')
- end
+ Tasks::Gitlab::Backup.restore_task('lfs')
end
end
namespace :terraform_state do
task create: :gitlab_environment do
- lock do
- Backup::Manager.new(progress).run_create_task('terraform_state')
- end
+ Tasks::Gitlab::Backup.create_task('terraform_state')
end
task restore: :gitlab_environment do
- lock do
- Backup::Manager.new(progress).run_restore_task('terraform_state')
- end
+ Tasks::Gitlab::Backup.restore_task('terraform_state')
end
end
namespace :registry do
task create: :gitlab_environment do
- lock do
- Backup::Manager.new(progress).run_create_task('registry')
- end
+ Tasks::Gitlab::Backup.create_task('registry')
end
task restore: :gitlab_environment do
- lock do
- Backup::Manager.new(progress).run_restore_task('registry')
- end
+ Tasks::Gitlab::Backup.restore_task('registry')
end
end
namespace :packages do
task create: :gitlab_environment do
- lock do
- Backup::Manager.new(progress).run_create_task('packages')
- end
+ Tasks::Gitlab::Backup.create_task('packages')
end
task restore: :gitlab_environment do
- lock do
- Backup::Manager.new(progress).run_restore_task('packages')
- end
- end
- end
-
- def progress
- if ENV['CRON']
- # We need an object we can say 'puts' and 'print' to; let's use a
- # StringIO.
- require 'stringio'
- StringIO.new
- else
- $stdout
- end
- end
-
- def lock
- File.open(PID_FILE, File::RDWR | File::CREAT, 0644) do |f|
- f.flock(File::LOCK_EX)
-
- unless f.read.empty?
- # There is a PID inside so the process fails
- progress.puts(<<~HEREDOC.color(:red))
- Backup and restore in progress:
- There is a backup and restore task in progress. Please, try to run the current task once the previous one ends.
- If there is no other process running, please remove the PID file manually: rm #{PID_FILE}
- HEREDOC
-
- exit 1
- end
-
- f.write(PID)
- f.flush
- ensure
- f.flock(File::LOCK_UN)
- end
-
- begin
- yield
- ensure
- progress.puts "#{Time.now} " + "-- Deleting backup and restore lock file".color(:blue)
- File.delete(PID_FILE)
+ Tasks::Gitlab::Backup.restore_task('packages')
end
end
end
diff --git a/lib/tasks/gitlab/bulk_add_permission.rake b/lib/tasks/gitlab/bulk_add_permission.rake
index a903c743ea2..240b808baf3 100644
--- a/lib/tasks/gitlab/bulk_add_permission.rake
+++ b/lib/tasks/gitlab/bulk_add_permission.rake
@@ -6,21 +6,21 @@ namespace :gitlab do
task all_users_to_all_projects: :environment do |t, args|
user_ids = User.where(admin: false).pluck(:id)
admin_ids = User.where(admin: true).pluck(:id)
- project_ids = Project.pluck(:id)
+ projects = Project.all
- puts "Importing #{user_ids.size} users into #{project_ids.size} projects"
- ProjectMember.add_members_to_projects(project_ids, user_ids, ProjectMember::DEVELOPER)
+ puts "Importing #{user_ids.size} users into #{projects.size} projects"
+ Members::Projects::CreatorService.add_members(projects, user_ids, ProjectMember::DEVELOPER)
- puts "Importing #{admin_ids.size} admins into #{project_ids.size} projects"
- ProjectMember.add_members_to_projects(project_ids, admin_ids, ProjectMember::MAINTAINER)
+ puts "Importing #{admin_ids.size} admins into #{projects.size} projects"
+ Members::Projects::CreatorService.add_members(projects, admin_ids, ProjectMember::MAINTAINER)
end
desc "GitLab | Import | Add a specific user to all projects (as a developer)"
task :user_to_projects, [:email] => :environment do |t, args|
user = User.find_by(email: args.email)
- project_ids = Project.pluck(:id)
- puts "Importing #{user.email} users into #{project_ids.size} projects"
- ProjectMember.add_members_to_projects(project_ids, Array.wrap(user.id), ProjectMember::DEVELOPER)
+ projects = Project.all
+ puts "Importing #{user.email} users into #{projects.size} projects"
+ Members::Projects::CreatorService.add_members(projects, Array.wrap(user.id), ProjectMember::DEVELOPER)
end
desc "GitLab | Import | Add all users to all groups (admin users are added as owners)"
diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake
index c4dc7b938cc..34ccce3ba2f 100644
--- a/lib/tasks/gitlab/db.rake
+++ b/lib/tasks/gitlab/db.rake
@@ -4,6 +4,7 @@ databases = ActiveRecord::Tasks::DatabaseTasks.setup_initial_database_yaml
def each_database(databases, include_geo: false)
ActiveRecord::Tasks::DatabaseTasks.for_each(databases) do |database|
+ next if database == 'embedding'
next if !include_geo && database == 'geo'
yield database
@@ -60,13 +61,18 @@ namespace :gitlab do
# In PostgreSQLAdapter, data_sources returns both views and tables, so use tables instead
tables = connection.tables
+ # Views that are dependencies to PG_EXTENSION (like pg_stat_statements) should be ignored
+ ignored_views = Gitlab::Database::PgDepend.using_connection(connection) do
+ Gitlab::Database::PgDepend.from_pg_extension('VIEW').pluck('relname')
+ end
+
# Removes the entry from the array
tables.delete 'schema_migrations'
# Truncate schema_migrations to ensure migrations re-run
connection.execute('TRUNCATE schema_migrations') if connection.table_exists? 'schema_migrations'
# Drop any views
- connection.views.each do |view|
+ (connection.views - ignored_views).each do |view|
connection.execute("DROP VIEW IF EXISTS #{connection.quote_table_name(view)} CASCADE")
end
@@ -109,9 +115,11 @@ namespace :gitlab do
load_database = connection.tables.count <= 1
if load_database
+ puts "Running db:schema:load#{database_name} rake task"
Gitlab::Database.add_post_migrate_path_to_rails(force: true)
Rake::Task["db:schema:load#{database_name}"].invoke
else
+ puts "Running db:migrate#{database_name} rake task"
Rake::Task["db:migrate#{database_name}"].invoke
end
@@ -131,7 +139,7 @@ namespace :gitlab do
end
end
- desc 'This adjusts and cleans db/structure.sql - it runs after db:structure:dump'
+ desc 'This adjusts and cleans db/structure.sql - it runs after db:schema:dump'
task :clean_structure_sql do |task_name|
ActiveRecord::Base.configurations.configs_for(env_name: ActiveRecord::Tasks::DatabaseTasks.env).each do |db_config|
structure_file = ActiveRecord::Tasks::DatabaseTasks.dump_filename(db_config.name)
@@ -147,26 +155,13 @@ namespace :gitlab do
Rake::Task[task_name].reenable
end
- # Inform Rake that custom tasks should be run every time rake db:structure:dump is run
- #
- # Rails 6.1 deprecates db:structure:dump in favor of db:schema:dump
- Rake::Task['db:structure:dump'].enhance do
- Rake::Task['gitlab:db:clean_structure_sql'].invoke
- end
-
# Inform Rake that custom tasks should be run every time rake db:schema:dump is run
Rake::Task['db:schema:dump'].enhance do
Rake::Task['gitlab:db:clean_structure_sql'].invoke
end
ActiveRecord::Tasks::DatabaseTasks.for_each(databases) do |name|
- # Inform Rake that custom tasks should be run every time rake db:structure:dump is run
- #
- # Rails 6.1 deprecates db:structure:dump in favor of db:schema:dump
- Rake::Task["db:structure:dump:#{name}"].enhance do
- Rake::Task['gitlab:db:clean_structure_sql'].invoke
- end
-
+ # Inform Rake that custom tasks should be run every time rake db:schema:dump is run
Rake::Task["db:schema:dump:#{name}"].enhance do
Rake::Task['gitlab:db:clean_structure_sql'].invoke
end
@@ -313,6 +308,36 @@ namespace :gitlab do
end
end
+ namespace :validate_async_constraints do
+ each_database(databases) do |database_name|
+ task database_name, [:pick] => :environment do |_, args|
+ args.with_defaults(pick: 2)
+
+ if Feature.disabled?(:database_async_foreign_key_validation, type: :ops)
+ puts <<~NOTE.color(:yellow)
+ Note: database async foreign key validation feature is currently disabled.
+
+ Enable with: Feature.enable(:database_async_foreign_key_validation)
+ NOTE
+ exit
+ end
+
+ Gitlab::Database::EachDatabase.each_database_connection(only: database_name) do
+ Gitlab::Database::AsyncConstraints.validate_pending_entries!(how_many: args[:pick].to_i)
+ end
+ end
+ end
+
+ task :all, [:pick] => :environment do |_, args|
+ default_pick = Gitlab.dev_or_test_env? ? 1000 : 2
+ args.with_defaults(pick: default_pick)
+
+ each_database(databases) do |database_name|
+ Rake::Task["gitlab:db:validate_async_constraints:#{database_name}"].invoke(args[:pick])
+ end
+ end
+ end
+
desc 'Check if there have been user additions to the database'
task active: :environment do
if ActiveRecord::Base.connection.migration_context.needs_migration?
@@ -320,11 +345,7 @@ namespace :gitlab do
exit 1
end
- # A list of projects that GitLab creates automatically on install/upgrade
- # gc = Gitlab::CurrentSettings.current_application_settings
- seed_projects = [Gitlab::CurrentSettings.current_application_settings.self_monitoring_project]
-
- if (Project.count - seed_projects.count { |x| !x.nil? }).eql?(0)
+ if Project.count.eql?(0)
puts "No user created projects. Database not active"
exit 1
end
@@ -427,21 +448,61 @@ namespace :gitlab do
end
end
+ namespace :schema_checker do
+ # TODO: Remove `test_replication` after PG 14 upgrade is finished
+ # https://gitlab.com/gitlab-com/gl-infra/db-migration/-/merge_requests/406#note_1369214728
+ IGNORED_TABLES = %w[test_replication].freeze
+ IGNORED_TRIGGERS = ['gitlab_schema_write_trigger_for_'].freeze
+
+ desc 'Checks schema inconsistencies'
+ task run: :environment do
+ database_model = Gitlab::Database.database_base_models[Gitlab::Database::MAIN_DATABASE_NAME]
+ database = Gitlab::Database::SchemaValidation::Database.new(database_model.connection)
+
+ stucture_sql_path = Rails.root.join('db/structure.sql')
+ structure_sql = Gitlab::Database::SchemaValidation::StructureSql.new(stucture_sql_path)
+
+ filter = Gitlab::Database::SchemaValidation::InconsistencyFilter.new(IGNORED_TABLES, IGNORED_TRIGGERS)
+
+ inconsistencies =
+ Gitlab::Database::SchemaValidation::Runner.new(structure_sql, database).execute.filter_map(&filter)
+
+ gitlab_url = 'gitlab-org/gitlab'
+
+ inconsistencies.each do |inconsistency|
+ Gitlab::Database::SchemaValidation::TrackInconsistency.new(
+ inconsistency,
+ Project.find_by_full_path(gitlab_url),
+ User.support_bot
+ ).execute
+
+ puts inconsistency.inspect
+ end
+ end
+ end
+
namespace :dictionary do
- DB_DOCS_PATH = File.join(Rails.root, 'db', 'docs')
- EE_DICTIONARY_PATH = File.join(Rails.root, 'ee', 'db', 'docs')
+ DB_DOCS_PATH = Rails.root.join('db', 'docs')
desc 'Generate database docs yaml'
task generate: :environment do
+ next if Gitlab.jh?
+
FileUtils.mkdir_p(DB_DOCS_PATH)
- FileUtils.mkdir_p(EE_DICTIONARY_PATH) if Gitlab.ee?
+
+ if Gitlab.ee?
+ Gitlab::Database::EE_DATABASES_NAME_TO_DIR.each do |_, ee_db_dir|
+ FileUtils.mkdir_p(Rails.root.join(ee_db_dir, 'docs'))
+ end
+ end
Rails.application.eager_load!
version = Gem::Version.new(File.read('VERSION'))
- milestone = version.release.segments[0..1].join('.')
+ milestone = version.release.segments.first(2).join('.')
classes = {}
+ ignored_tables = %w[p_ci_builds]
Gitlab::Database.database_base_models.each do |_, model_class|
tables = model_class.connection.tables
@@ -459,10 +520,12 @@ namespace :gitlab do
.reject(&:abstract_class)
.reject { |c| c.name =~ /^(?:EE::)?Gitlab::(?:BackgroundMigration|DatabaseImporters)::/ }
.reject { |c| c.name =~ /^HABTM_/ }
+ .reject { |c| c < Gitlab::Database::Migration[1.0]::MigrationRecord }
.each { |c| classes[c.table_name] << c.name if classes.has_key?(c.table_name) }
sources.each do |source_name|
next if source_name.start_with?('_test_') # Ignore test tables
+ next if ignored_tables.include?(source_name)
database = model_class.connection_db_config.name
file = dictionary_file_path(source_name, views, database)
@@ -488,7 +551,7 @@ namespace :gitlab do
end
if existing_metadata['classes'] && existing_metadata['classes'].sort != table_metadata['classes'].sort
- existing_metadata['classes'] = table_metadata['classes']
+ existing_metadata['classes'] = (existing_metadata['classes'] + table_metadata['classes']).uniq.sort
outdated = true
end
@@ -511,7 +574,11 @@ namespace :gitlab do
def dictionary_file_path(source_name, views, database)
sub_directory = views.include?(source_name) ? 'views' : ''
- path = database == 'geo' ? EE_DICTIONARY_PATH : DB_DOCS_PATH
+ path = if Gitlab.ee? && Gitlab::Database::EE_DATABASES_NAME_TO_DIR.key?(database.to_s)
+ Rails.root.join(Gitlab::Database::EE_DATABASES_NAME_TO_DIR[database.to_s], 'docs')
+ else
+ DB_DOCS_PATH
+ end
File.join(path, sub_directory, "#{source_name}.yml")
end
diff --git a/lib/tasks/gitlab/db/decomposition/connection_status.rake b/lib/tasks/gitlab/db/decomposition/connection_status.rake
new file mode 100644
index 00000000000..a7230126158
--- /dev/null
+++ b/lib/tasks/gitlab/db/decomposition/connection_status.rake
@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+namespace :gitlab do
+ namespace :db do
+ namespace :decomposition do
+ desc 'Check if PostgreSQL max_connections needs to be increased'
+ task connection_status: :environment do
+ if Gitlab::Database.database_base_models.has_key?(:ci)
+ puts "GitLab database already running on two connections"
+ next
+ end
+
+ sql = <<~SQL
+ select q1.active, q2.max from
+ (select count(*) as active from pg_stat_activity) q1,
+ (select setting::int as max from pg_settings where name='max_connections') q2
+ SQL
+
+ active, max = ApplicationRecord.connection.select_one(sql).values
+
+ puts "Currently using #{active} connections out of #{max} max_connections,"
+
+ if active / max.to_f > 0.5
+ puts <<~ADVISE_INCREASE
+ which may run out when you switch to two database connections.
+
+ Consider increasing PostgreSQL 'max_connections' setting.
+ Depending on the installation method, there are different ways to
+ increase that setting. Please consult the GitLab documentation.
+ ADVISE_INCREASE
+ else
+ puts "which is enough for running GitLab using two database connections."
+ end
+ end
+ end
+ end
+end
diff --git a/lib/tasks/gitlab/docs/redirect.rake b/lib/tasks/gitlab/docs/redirect.rake
index 2d234fcdb36..35e0d083210 100644
--- a/lib/tasks/gitlab/docs/redirect.rake
+++ b/lib/tasks/gitlab/docs/redirect.rake
@@ -1,12 +1,13 @@
# frozen_string_literal: true
-require 'date'
-require 'pathname'
-require "yaml"
#
# https://docs.gitlab.com/ee/development/documentation/#move-or-rename-a-page
#
namespace :gitlab do
+ require 'date'
+ require 'pathname'
+ require 'yaml'
+
namespace :docs do
desc 'GitLab | Docs | Create a doc redirect'
task :redirect, [:old_path, :new_path] do |_, args|
diff --git a/lib/tasks/gitlab/feature_categories.rake b/lib/tasks/gitlab/feature_categories.rake
index cecfaf3cb36..db496012158 100644
--- a/lib/tasks/gitlab/feature_categories.rake
+++ b/lib/tasks/gitlab/feature_categories.rake
@@ -15,7 +15,7 @@ namespace :gitlab do
hash[feature_category] << {
klass: controller.to_s,
action: action,
- source_location: source_location(controller, action)
+ source_location: src_location(controller, action)
}
end
@@ -28,7 +28,7 @@ namespace :gitlab do
hash[feature_category] << {
klass: klass.to_s,
action: path,
- source_location: source_location(klass)
+ source_location: src_location(klass)
}
end
@@ -40,7 +40,7 @@ namespace :gitlab do
hash[feature_category] ||= []
hash[feature_category] << {
klass: worker.klass.name,
- source_location: source_location(worker.klass.name)
+ source_location: src_location(worker.klass.name)
}
end
@@ -60,7 +60,14 @@ namespace :gitlab do
'database_tables' => database_tables)
end
- def source_location(klass, method = nil)
+ private
+
+ # Source location of the trace
+ # @param [Class] klass
+ # @param [Method,UnboundMethod] method
+ # @note This method was named `source_location` but this name shadowed Binding#source_location
+ # @note This method was made private as it is not being used elsewhere
+ def src_location(klass, method = nil)
file, line =
if method && klass.method_defined?(method)
klass.instance_method(method).source_location
diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake
index e814d59aaf9..4c19d94f4a1 100644
--- a/lib/tasks/gitlab/gitaly.rake
+++ b/lib/tasks/gitlab/gitaly.rake
@@ -26,17 +26,8 @@ Usage: rake "gitlab:gitaly:install[/installation/dir,/storage/path]")
Gitlab::SetupHelper::Gitaly.create_configuration(args.dir, storage_paths)
Dir.chdir(args.dir) do
- Bundler.with_original_env do
- env = { "RUBYOPT" => nil, "BUNDLE_GEMFILE" => nil }
-
- if Rails.env.test?
- env["GEM_HOME"] = Bundler.bundle_path.to_s
- env["BUNDLE_DEPLOYMENT"] = 'false'
- end
-
- output, status = Gitlab::Popen.popen([make_cmd, 'clean-build', 'all'], nil, env)
- raise "Gitaly failed to compile: #{output}" unless status&.zero?
- end
+ output, status = Gitlab::Popen.popen([make_cmd, 'clean', 'all'])
+ raise "Gitaly failed to compile: #{output}" unless status&.zero?
end
end
diff --git a/lib/tasks/gitlab/graphql.rake b/lib/tasks/gitlab/graphql.rake
index a05b749a60e..8a677ff4677 100644
--- a/lib/tasks/gitlab/graphql.rake
+++ b/lib/tasks/gitlab/graphql.rake
@@ -2,10 +2,10 @@
return if Rails.env.production?
-require 'graphql/rake_task'
-require_relative '../../../tooling/graphql/docs/renderer'
-
namespace :gitlab do
+ require 'graphql/rake_task'
+ require_relative '../../../tooling/graphql/docs/renderer'
+
OUTPUT_DIR = Rails.root.join("doc/api/graphql/reference")
TEMP_SCHEMA_DIR = Rails.root.join('tmp/tests/graphql')
TEMPLATES_DIR = 'tooling/graphql/docs/templates/'
@@ -71,6 +71,12 @@ namespace :gitlab do
desc 'GitLab | GraphQL | Validate queries'
task validate: [:environment, :enable_feature_flags] do |t, args|
+ class GenerousTimeoutSchema < GitlabSchema # rubocop:disable Gitlab/NamespacedClass
+ validate_timeout 1.second
+ end
+
+ puts "Validating GraphQL queries. Validation timeout set to #{GenerousTimeoutSchema.validate_timeout} second(s)"
+
queries = if args.to_a.present?
args.to_a.flat_map { |path| Gitlab::Graphql::Queries.find(path) }
else
@@ -78,7 +84,7 @@ namespace :gitlab do
end
failed = queries.flat_map do |defn|
- summary, errs = defn.validate(GitlabSchema)
+ summary, errs = defn.validate(GenerousTimeoutSchema)
case summary
when :client_query
@@ -86,8 +92,10 @@ namespace :gitlab do
else
warn("#{'OK'.color(:green)} #{defn.file}") if errs.empty?
errs.each do |err|
+ path_info = "(at #{err.path.join('.')})" if err.path
+
warn(<<~MSG)
- #{'ERROR'.color(:red)} #{defn.file}: #{err.message} (at #{err.path.join('.')})
+ #{'ERROR'.color(:red)} #{defn.file}: #{err.message} #{path_info}
MSG
end
end
diff --git a/lib/tasks/gitlab/import.rake b/lib/tasks/gitlab/import.rake
deleted file mode 100644
index bf0ba40fb31..00000000000
--- a/lib/tasks/gitlab/import.rake
+++ /dev/null
@@ -1,24 +0,0 @@
-# frozen_string_literal: true
-
-namespace :gitlab do
- namespace :import do
- # How to use:
- #
- # 1. copy the bare repos to a specific path that contain the group or subgroups structure as folders
- # 2. run: bundle exec rake gitlab:import:repos[/path/to/repos] RAILS_ENV=production
- #
- # Notes:
- # * The project owner will set to the first administator of the system
- # * Existing projects will be skipped
- desc "GitLab | Import | Import bare repositories from repositories -> storages into GitLab project instance"
- task :repos, [:import_path] => :environment do |_t, args|
- unless args.import_path
- puts 'Please specify an import path that contains the repositories'.color(:red)
-
- exit 1
- end
-
- Gitlab::BareRepositoryImport::Importer.execute(args.import_path)
- end
- end
-end
diff --git a/lib/tasks/gitlab/import_export/import.rake b/lib/tasks/gitlab/import_export/import.rake
index fc727eda380..2c219717535 100644
--- a/lib/tasks/gitlab/import_export/import.rake
+++ b/lib/tasks/gitlab/import_export/import.rake
@@ -30,10 +30,12 @@ namespace :gitlab do
end
task = Gitlab::ImportExport::Project::ImportTask.new(
- namespace_path: args.namespace_path,
- project_path: args.project_path,
- username: args.username,
- file_path: args.archive_path,
+ {
+ namespace_path: args.namespace_path,
+ project_path: args.project_path,
+ username: args.username,
+ file_path: args.archive_path
+ },
logger: logger
)
diff --git a/lib/tasks/gitlab/lfs/migrate.rake b/lib/tasks/gitlab/lfs/migrate.rake
index 47f9e1dfb32..19de45dca17 100644
--- a/lib/tasks/gitlab/lfs/migrate.rake
+++ b/lib/tasks/gitlab/lfs/migrate.rake
@@ -1,9 +1,9 @@
# frozen_string_literal: true
-require 'logger'
-
desc "GitLab | LFS | Migrate LFS objects to remote storage"
namespace :gitlab do
+ require 'logger'
+
namespace :lfs do
task migrate: :environment do
logger = Logger.new($stdout)
diff --git a/lib/tasks/gitlab/metrics_exporter.rake b/lib/tasks/gitlab/metrics_exporter.rake
index d9dd80b8eeb..70719648fc5 100644
--- a/lib/tasks/gitlab/metrics_exporter.rake
+++ b/lib/tasks/gitlab/metrics_exporter.rake
@@ -1,8 +1,9 @@
# frozen_string_literal: true
-require_relative Rails.root.join('metrics_server', 'dependencies')
-require_relative Rails.root.join('metrics_server', 'metrics_server')
namespace :gitlab do
+ require_relative Rails.root.join('metrics_server', 'dependencies')
+ require_relative Rails.root.join('metrics_server', 'metrics_server')
+
namespace :metrics_exporter do
REPO = 'https://gitlab.com/gitlab-org/gitlab-metrics-exporter.git'
diff --git a/lib/tasks/gitlab/openapi.rake b/lib/tasks/gitlab/openapi.rake
index dee365de11c..c9aec6a112a 100644
--- a/lib/tasks/gitlab/openapi.rake
+++ b/lib/tasks/gitlab/openapi.rake
@@ -1,13 +1,13 @@
# frozen_string_literal: true
-require 'logger'
-
if Rails.env.development?
require 'grape-swagger/rake/oapi_tasks'
GrapeSwagger::Rake::OapiTasks.new('::API::API')
end
namespace :gitlab do
+ require 'logger'
+
namespace :openapi do
task :validate do
raise 'This task can only be run in the development environment' unless Rails.env.development?
diff --git a/lib/tasks/gitlab/packages/events.rake b/lib/tasks/gitlab/packages/events.rake
index a5b801ff62d..1234ba039a3 100644
--- a/lib/tasks/gitlab/packages/events.rake
+++ b/lib/tasks/gitlab/packages/events.rake
@@ -1,7 +1,5 @@
# frozen_string_literal: true
-require 'logger'
-
desc "GitLab | Packages | Events | Generate hll counter events file for packages"
namespace :gitlab do
namespace :packages do
@@ -40,18 +38,16 @@ namespace :gitlab do
private
def event_pairs
- Packages::Event.event_types.keys.product(Packages::Event::EVENT_SCOPES.keys)
+ Packages::Event::EVENT_TYPES.product(Packages::Event::EVENT_SCOPES.keys)
end
def generate_unique_events_list
events = event_pairs.each_with_object([]) do |(event_type, event_scope), events|
- Packages::Event.originator_types.keys.excluding('guest').each do |originator_type|
+ Packages::Event::ORIGINATOR_TYPES.excluding(:guest).each do |originator_type|
events_definition = Packages::Event.unique_counters_for(event_scope, event_type, originator_type).map do |event_name|
{
"name" => event_name,
- "category" => "#{originator_type}_packages",
- "aggregation" => "weekly",
- "redis_slot" => "package"
+ "aggregation" => "weekly"
}
end
@@ -64,7 +60,7 @@ namespace :gitlab do
def counter_events_list
counters = event_pairs.flat_map do |event_type, event_scope|
- Packages::Event.originator_types.keys.flat_map do |originator_type|
+ Packages::Event::ORIGINATOR_TYPES.flat_map do |originator_type|
Packages::Event.counters_for(event_scope, event_type, originator_type)
end
end
diff --git a/lib/tasks/gitlab/packages/migrate.rake b/lib/tasks/gitlab/packages/migrate.rake
index 1c28f4308a2..737788583ae 100644
--- a/lib/tasks/gitlab/packages/migrate.rake
+++ b/lib/tasks/gitlab/packages/migrate.rake
@@ -1,11 +1,11 @@
# frozen_string_literal: true
-require 'logger'
-
desc "GitLab | Packages | Migrate packages files to remote storage"
namespace :gitlab do
namespace :packages do
task migrate: :environment do
+ require 'logger'
+
logger = Logger.new($stdout)
logger.info('Starting transfer of package files to object storage')
diff --git a/lib/tasks/gitlab/pages.rake b/lib/tasks/gitlab/pages.rake
index e6fde28e38f..ecfb163f284 100644
--- a/lib/tasks/gitlab/pages.rake
+++ b/lib/tasks/gitlab/pages.rake
@@ -1,11 +1,11 @@
# frozen_string_literal: true
-require 'logger'
-
namespace :gitlab do
namespace :pages do
namespace :deployments do
task migrate_to_object_storage: :gitlab_environment do
+ require 'logger'
+
logger = Logger.new($stdout)
helper = Gitlab::LocalAndRemoteStorageMigration::PagesDeploymentMigrater.new(logger)
diff --git a/lib/tasks/gitlab/refresh_project_statistics_build_artifacts_size.rake b/lib/tasks/gitlab/refresh_project_statistics_build_artifacts_size.rake
index 203d500b616..9ed38a86bbd 100644
--- a/lib/tasks/gitlab/refresh_project_statistics_build_artifacts_size.rake
+++ b/lib/tasks/gitlab/refresh_project_statistics_build_artifacts_size.rake
@@ -1,19 +1,21 @@
# frozen_string_literal: true
-require 'httparty'
-require 'csv'
-
namespace :gitlab do
- desc "GitLab | Refresh build artifacts size project statistics for given list of Project IDs from remote CSV"
+ desc "GitLab | Refresh build artifacts size project statistics for given list of Project IDs from CSV"
BUILD_ARTIFACTS_SIZE_REFRESH_ENQUEUE_BATCH_SIZE = 500
- task :refresh_project_statistics_build_artifacts_size, [:csv_url] => :environment do |_t, args|
- csv_url = args.csv_url
+ task :refresh_project_statistics_build_artifacts_size, [:csv_path] => :environment do |_t, args|
+ require 'httparty'
+ require 'csv'
+
+ csv_path = args.csv_path
- # rubocop: disable Gitlab/HTTParty
- body = HTTParty.get(csv_url)
- # rubocop: enable Gitlab/HTTParty
+ body = if csv_path.start_with?('http')
+ HTTParty.get(csv_path) # rubocop: disable Gitlab/HTTParty
+ else
+ File.read(csv_path)
+ end
table = CSV.parse(body.to_s, headers: true)
project_ids = table['PROJECT_ID']
diff --git a/lib/tasks/gitlab/seed/ci_variables_group.rake b/lib/tasks/gitlab/seed/ci_variables_group.rake
new file mode 100644
index 00000000000..d58923eab0e
--- /dev/null
+++ b/lib/tasks/gitlab/seed/ci_variables_group.rake
@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+# Seed group with CI variables
+#
+# @param name - name of the group to add CI variables to
+# @param seed_count - total number of CI variables to create (default: 10)
+# @param environment_scope - environment scope of the variable (default: '*')
+# If "unique", it will create a unique environment_scope per variable.
+# @param prefix - prefix of the variable key (default: 'GROUP_VAR_')
+#
+# @example
+# bundle exec rake "gitlab:seed:ci_variables_group[kitchen-sink, 5, unique]"
+#
+namespace :gitlab do
+ namespace :seed do
+ desc 'Seed group with CI Variables'
+ task :ci_variables_group,
+ [:name, :seed_count, :environment_scope, :prefix] => :gitlab_environment do |_t, args|
+ Gitlab::Seeders::Ci::VariablesGroupSeeder.new(
+ name: args.name,
+ seed_count: args.seed_count&.to_i,
+ prefix: args&.prefix,
+ environment_scope: args&.environment_scope
+ ).seed
+ puts "Task finished!"
+ end
+ end
+end
diff --git a/lib/tasks/gitlab/seed/ci_variables_instance.rake b/lib/tasks/gitlab/seed/ci_variables_instance.rake
new file mode 100644
index 00000000000..15a101a1cf1
--- /dev/null
+++ b/lib/tasks/gitlab/seed/ci_variables_instance.rake
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+# Seed instance with CI variables
+#
+# @param seed_count - total number of CI variables to create (default: 10)
+# @param prefix - prefix of the variable key (default: 'INSTANCE_VAR_')
+#
+# @example
+# bundle exec rake "gitlab:seed:ci_variables_instance[5, INSTANCE_TEST_]"
+#
+namespace :gitlab do
+ namespace :seed do
+ desc 'Seed instance with CI Variables'
+ task :ci_variables_instance,
+ [:seed_count, :prefix] => :gitlab_environment do |_t, args|
+ Gitlab::Seeders::Ci::VariablesInstanceSeeder.new(
+ seed_count: args.seed_count&.to_i,
+ prefix: args&.prefix
+ ).seed
+ puts "Task finished!"
+ end
+ end
+end
diff --git a/lib/tasks/gitlab/seed/ci_variables_project.rake b/lib/tasks/gitlab/seed/ci_variables_project.rake
new file mode 100644
index 00000000000..64f8b8d2a86
--- /dev/null
+++ b/lib/tasks/gitlab/seed/ci_variables_project.rake
@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+# Seed project with CI variables
+#
+# @param project_path - path of the project to add CI variables to
+# @param seed_count - total number of CI variables to create (default: 10)
+# @param environment_scope - environment scope of the variable (default: '*')
+# If "unique", it will create a unique environment_scope per variable.
+# @param prefix - prefix of the variable key (default: 'VAR_')
+#
+# @example
+# bundle exec rake "gitlab:seed:ci_variables_project[root/paper, 5, production, prod_var_]"
+#
+namespace :gitlab do
+ namespace :seed do
+ desc 'Seed project with CI Variables'
+ task :ci_variables_project,
+ [:project_path, :seed_count, :environment_scope, :prefix] => :gitlab_environment do |_t, args|
+ Gitlab::Seeders::Ci::VariablesProjectSeeder.new(
+ project_path: args.project_path,
+ seed_count: args.seed_count&.to_i,
+ prefix: args&.prefix,
+ environment_scope: args&.environment_scope
+ ).seed
+ puts "Task finished!"
+ end
+ end
+end
diff --git a/lib/tasks/gitlab/seed/project_environments.rake b/lib/tasks/gitlab/seed/project_environments.rake
new file mode 100644
index 00000000000..8557c130297
--- /dev/null
+++ b/lib/tasks/gitlab/seed/project_environments.rake
@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+# Seed project with environments
+#
+# @param project_path - path of the project to add environments to
+# @param seed_count - total number of environments to create (default: 10)
+# @param prefix - prefix used for the environment name (default: 'ENV_')
+#
+# @example
+# bundle exec rake "gitlab:seed:project_environments[root/paper, 5, staging_]"
+#
+namespace :gitlab do
+ namespace :seed do
+ desc 'Seed project with environments'
+ task :project_environments, [:project_path, :seed_count, :prefix] => :gitlab_environment do |_t, args|
+ Gitlab::Seeders::ProjectEnvironmentSeeder.new(
+ project_path: args.project_path,
+ seed_count: args.seed_count&.to_i,
+ prefix: args&.prefix
+ ).seed
+ puts "Task finished!"
+ end
+ end
+end
diff --git a/lib/tasks/gitlab/seed/runner_fleet.rake b/lib/tasks/gitlab/seed/runner_fleet.rake
index c0b79269c75..784451226b7 100644
--- a/lib/tasks/gitlab/seed/runner_fleet.rake
+++ b/lib/tasks/gitlab/seed/runner_fleet.rake
@@ -28,10 +28,12 @@ namespace :gitlab do
runner_count: args.runner_count&.to_i
).seed
- Gitlab::Seeders::Ci::Runner::RunnerFleetPipelineSeeder.new(
- projects_to_runners: projects_to_runners,
- job_count: args.job_count&.to_i
- ).seed
+ if projects_to_runners
+ Gitlab::Seeders::Ci::Runner::RunnerFleetPipelineSeeder.new(
+ projects_to_runners: projects_to_runners,
+ job_count: args.job_count&.to_i
+ ).seed
+ end
end
puts "Seed finished. Timings: #{timings}"
diff --git a/lib/tasks/gitlab/setup.rake b/lib/tasks/gitlab/setup.rake
index 006dfad3a95..38c5902702c 100644
--- a/lib/tasks/gitlab/setup.rake
+++ b/lib/tasks/gitlab/setup.rake
@@ -33,6 +33,7 @@ namespace :gitlab do
Rake::Task["dev:terminate_all_connections"].invoke unless Rails.env.production?
Rake::Task["db:reset"].invoke
+ Rake::Task["gitlab:db:lock_writes"].invoke
Rake::Task["db:seed_fu"].invoke
rescue Gitlab::TaskAbortedByUserError
puts "Quitting...".color(:red)
diff --git a/lib/tasks/gitlab/terraform/migrate.rake b/lib/tasks/gitlab/terraform/migrate.rake
index 99e33011cf5..3347b823376 100644
--- a/lib/tasks/gitlab/terraform/migrate.rake
+++ b/lib/tasks/gitlab/terraform/migrate.rake
@@ -1,11 +1,11 @@
# frozen_string_literal: true
-require 'logger'
-
desc "GitLab | Terraform | Migrate Terraform states to remote storage"
namespace :gitlab do
namespace :terraform_states do
task migrate: :environment do
+ require 'logger'
+
logger = Logger.new($stdout)
logger.info('Starting transfer of Terraform states to object storage')
diff --git a/lib/tasks/gitlab/tw/codeowners.rake b/lib/tasks/gitlab/tw/codeowners.rake
index 77f5eb87725..b4b34581f43 100644
--- a/lib/tasks/gitlab/tw/codeowners.rake
+++ b/lib/tasks/gitlab/tw/codeowners.rake
@@ -1,10 +1,10 @@
# frozen_string_literal: true
-require 'yaml'
-
namespace :tw do
desc 'Generates a list of codeowners for documentation pages.'
task :codeowners do
+ require 'yaml'
+
CodeOwnerRule = Struct.new(:category, :writer)
DocumentOwnerMapping = Struct.new(:path, :writer) do
def writer_owns_directory?(mappings)
@@ -19,67 +19,68 @@ namespace :tw do
end
CODE_OWNER_RULES = [
- CodeOwnerRule.new('Activation', '@phillipwells'),
- CodeOwnerRule.new('Acquisition', '@phillipwells'),
+ # CodeOwnerRule.new('Activation', ''),
+ # CodeOwnerRule.new('Acquisition', ''),
+ # CodeOwnerRule.new('AI Assisted', ''),
+ CodeOwnerRule.new('Analytics Instrumentation', '@lciutacu'),
CodeOwnerRule.new('Anti-Abuse', '@phillipwells'),
+ CodeOwnerRule.new('Application Performance', '@jglassman1'),
CodeOwnerRule.new('Authentication and Authorization', '@jglassman1'),
- CodeOwnerRule.new('Certify', '@msedlakjakubowski'),
+ # CodeOwnerRule.new('Billing and Subscription Management', ''),
CodeOwnerRule.new('Code Review', '@aqualls'),
CodeOwnerRule.new('Compliance', '@eread'),
- CodeOwnerRule.new('Commerce Integrations', '@drcatherinepope'),
CodeOwnerRule.new('Composition Analysis', '@rdickenson'),
- CodeOwnerRule.new('Configure', '@phillipwells'),
- CodeOwnerRule.new('Container Registry', '@dianalogan'),
+ CodeOwnerRule.new('Environments', '@phillipwells'),
+ CodeOwnerRule.new('Container Registry', '@marcel.amirault'),
CodeOwnerRule.new('Contributor Experience', '@eread'),
- CodeOwnerRule.new('Conversion', '@kpaizee'),
CodeOwnerRule.new('Database', '@aqualls'),
+ # CodeOwnerRule.new('DataOps', ''),
+ # CodeOwnerRule.new('Delivery', ''),
CodeOwnerRule.new('Development', '@sselhorn'),
CodeOwnerRule.new('Distribution', '@axil'),
CodeOwnerRule.new('Distribution (Charts)', '@axil'),
- CodeOwnerRule.new('Distribution (Omnibus)', '@axil'),
+ CodeOwnerRule.new('Distribution (Omnibus)', '@eread'),
CodeOwnerRule.new('Documentation Guidelines', '@sselhorn'),
CodeOwnerRule.new('Dynamic Analysis', '@rdickenson'),
- CodeOwnerRule.new('Editor', '@ashrafkhamis'),
- CodeOwnerRule.new('Foundations', '@rdickenson'),
+ CodeOwnerRule.new('IDE', '@ashrafkhamis'),
+ CodeOwnerRule.new('Foundations', '@sselhorn'),
+ # CodeOwnerRule.new('Fulfillment Platform', ''),
CodeOwnerRule.new('Fuzz Testing', '@rdickenson'),
CodeOwnerRule.new('Geo', '@axil'),
CodeOwnerRule.new('Gitaly', '@eread'),
+ CodeOwnerRule.new('GitLab Dedicated', '@drcatherinepope'),
CodeOwnerRule.new('Global Search', '@ashrafkhamis'),
- CodeOwnerRule.new('Import', '@eread'),
+ CodeOwnerRule.new('Import and Integrate', '@eread @ashrafkhamis'),
CodeOwnerRule.new('Infrastructure', '@sselhorn'),
- CodeOwnerRule.new('Integrations', '@ashrafkhamis'),
- CodeOwnerRule.new('Knowledge', '@aqualls'),
- CodeOwnerRule.new('Application Performance', '@jglassman1'),
- CodeOwnerRule.new('Monitor', '@msedlakjakubowski'),
+ # CodeOwnerRule.new('Knowledge', ''),
+ # CodeOwnerRule.new('MLOps', '')
CodeOwnerRule.new('Observability', '@drcatherinepope'),
CodeOwnerRule.new('Optimize', '@lciutacu'),
- CodeOwnerRule.new('Package Registry', '@dianalogan'),
+ CodeOwnerRule.new('Organization', '@lciutacu'),
+ CodeOwnerRule.new('Package Registry', '@marcel.amirault'),
CodeOwnerRule.new('Pipeline Authoring', '@marcel.amirault'),
CodeOwnerRule.new('Pipeline Execution', '@drcatherinepope'),
- CodeOwnerRule.new('Pipeline Insights', '@marcel.amirault'),
- CodeOwnerRule.new('Portfolio Management', '@msedlakjakubowski'),
+ CodeOwnerRule.new('Pipeline Security', '@marcel.amirault'),
CodeOwnerRule.new('Product Analytics', '@lciutacu'),
- CodeOwnerRule.new('Product Intelligence', '@dianalogan'),
CodeOwnerRule.new('Product Planning', '@msedlakjakubowski'),
CodeOwnerRule.new('Project Management', '@msedlakjakubowski'),
CodeOwnerRule.new('Provision', '@fneill'),
CodeOwnerRule.new('Purchase', '@fneill'),
CodeOwnerRule.new('Redirect', 'Redirect'),
- CodeOwnerRule.new('Release', '@rdickenson'),
CodeOwnerRule.new('Respond', '@msedlakjakubowski'),
CodeOwnerRule.new('Runner', '@fneill'),
CodeOwnerRule.new('Runner SaaS', '@fneill'),
- CodeOwnerRule.new('Pods', '@jglassman1'),
- CodeOwnerRule.new('Security Policies', '@dianalogan'),
+ CodeOwnerRule.new('Security Policies', '@rdickenson'),
CodeOwnerRule.new('Source Code', '@aqualls'),
CodeOwnerRule.new('Static Analysis', '@rdickenson'),
CodeOwnerRule.new('Style Guide', '@sselhorn'),
+ CodeOwnerRule.new('Tenant Scale', '@lciutacu'),
CodeOwnerRule.new('Testing', '@eread'),
- CodeOwnerRule.new('Threat Insights', '@dianalogan'),
+ CodeOwnerRule.new('Threat Insights', '@rdickenson'),
CodeOwnerRule.new('Tutorials', '@kpaizee'),
- CodeOwnerRule.new('Utilization', '@fneill'),
- CodeOwnerRule.new('Vulnerability Research', '@dianalogan'),
- CodeOwnerRule.new('Organization', '@lciutacu')
+ # CodeOwnerRule.new('US Public Sector Services', ''),
+ CodeOwnerRule.new('Utilization', '@fneill')
+ # CodeOwnerRule.new('Vulnerability Research', '')
].freeze
ERRORS_EXCLUDED_FILES = [
diff --git a/lib/tasks/gitlab/usage_data.rake b/lib/tasks/gitlab/usage_data.rake
index 32db5e2dff6..fcbec4b0dba 100644
--- a/lib/tasks/gitlab/usage_data.rake
+++ b/lib/tasks/gitlab/usage_data.rake
@@ -88,8 +88,6 @@ namespace :gitlab do
def ci_template_event(event_name)
{
'name' => event_name,
- 'category' => 'ci_templates',
- 'redis_slot' => Gitlab::UsageDataCounters::CiTemplateUniqueCounter::REDIS_SLOT,
'aggregation' => 'weekly'
}
end
diff --git a/lib/tasks/gitlab/x509/update.rake b/lib/tasks/gitlab/x509/update.rake
index 7b7d15479bf..dc868ede05d 100644
--- a/lib/tasks/gitlab/x509/update.rake
+++ b/lib/tasks/gitlab/x509/update.rake
@@ -1,11 +1,11 @@
# frozen_string_literal: true
-require 'logger'
-
desc "GitLab | X509 | Update signatures when certificate store has changed"
namespace :gitlab do
namespace :x509 do
task update_signatures: :environment do
+ require 'logger'
+
update_certificates
end
diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake
index d2056338350..54aee5ff9f5 100644
--- a/lib/tasks/import.rake
+++ b/lib/tasks/import.rake
@@ -1,8 +1,5 @@
# frozen_string_literal: true
-require 'benchmark'
-require 'rainbow/ext/string'
-
class GithubImport
def self.run!(...)
new(...).run!
@@ -124,6 +121,9 @@ class GithubRepos
end
namespace :import do
+ require 'benchmark'
+ require 'rainbow/ext/string'
+
desc 'GitLab | Import | Import a GitHub project - Example: import:github[ToKeN,root,root/blah,my/github_repo] (optional my/github_repo)'
task :github, [:token, :gitlab_username, :project_path] => :environment do |_t, args|
abort 'Project path must be: namespace(s)/project_name'.color(:red) unless args.project_path.include?('/')
diff --git a/lib/tasks/tokens.rake b/lib/tasks/tokens.rake
index ff14ab51b49..81e24f4f7b6 100644
--- a/lib/tasks/tokens.rake
+++ b/lib/tasks/tokens.rake
@@ -1,11 +1,11 @@
# frozen_string_literal: true
-require_relative '../../app/models/concerns/token_authenticatable'
-require_relative '../../app/models/concerns/token_authenticatable_strategies/base'
-require_relative '../../app/models/concerns/token_authenticatable_strategies/insecure'
-require_relative '../../app/models/concerns/token_authenticatable_strategies/digest'
-
namespace :tokens do
+ require_relative '../../app/models/concerns/token_authenticatable'
+ require_relative '../../app/models/concerns/token_authenticatable_strategies/base'
+ require_relative '../../app/models/concerns/token_authenticatable_strategies/insecure'
+ require_relative '../../app/models/concerns/token_authenticatable_strategies/digest'
+
desc "Reset all GitLab incoming email tokens"
task reset_all_email: :environment do
reset_all_users_token(:reset_incoming_email_token!)
diff --git a/lib/uploaded_file.rb b/lib/uploaded_file.rb
index 36baf4a3cf8..0ad982dc127 100644
--- a/lib/uploaded_file.rb
+++ b/lib/uploaded_file.rb
@@ -7,6 +7,7 @@ require "fileutils"
class UploadedFile
InvalidPathError = Class.new(StandardError)
UnknownSizeError = Class.new(StandardError)
+ ALLOWED_KWARGS = %i[filename content_type sha256 remote_id size upload_duration sha1 md5].freeze
# The filename, *not* including the path, of the "uploaded" file
attr_reader :original_filename
@@ -17,12 +18,11 @@ class UploadedFile
# The content type of the "uploaded" file
attr_accessor :content_type
- attr_reader :remote_id
- attr_reader :sha256
- attr_reader :size
- attr_reader :upload_duration
+ attr_reader :remote_id, :sha256, :size, :upload_duration, :sha1, :md5
+
+ def initialize(path, **kwargs)
+ validate_kwargs(kwargs)
- def initialize(path, filename: nil, content_type: "application/octet-stream", sha256: nil, remote_id: nil, size: nil, upload_duration: nil)
if path.present?
raise InvalidPathError, "#{path} file does not exist" unless ::File.exist?(path)
@@ -30,23 +30,24 @@ class UploadedFile
@size = @tempfile.size
else
begin
- @size = Integer(size)
+ @size = Integer(kwargs[:size])
rescue ArgumentError, TypeError
raise UnknownSizeError, 'Unable to determine file size'
end
end
begin
- @upload_duration = Float(upload_duration)
+ @upload_duration = Float(kwargs[:upload_duration])
rescue ArgumentError, TypeError
@upload_duration = 0
end
- @content_type = content_type
- @original_filename = sanitize_filename(filename || path || '')
- @content_type = content_type
- @sha256 = sha256
- @remote_id = remote_id
+ @content_type = kwargs[:content_type] || 'application/octet-stream'
+ @original_filename = sanitize_filename(kwargs[:filename] || path || '')
+ @sha256 = kwargs[:sha256]
+ @sha1 = kwargs[:sha1]
+ @md5 = kwargs[:md5]
+ @remote_id = kwargs[:remote_id]
end
def self.from_params(params, upload_paths)
@@ -65,14 +66,16 @@ class UploadedFile
end
end
- UploadedFile.new(
+ new(
file_path,
filename: params['name'],
content_type: params['type'] || 'application/octet-stream',
sha256: params['sha256'],
remote_id: remote_id,
size: params['size'],
- upload_duration: params['upload_duration']
+ upload_duration: params['upload_duration'],
+ sha1: params['sha1'],
+ md5: params['md5']
).tap do |uploaded_file|
::Gitlab::Instrumentation::Uploads.track(uploaded_file)
end
@@ -111,4 +114,11 @@ class UploadedFile
def respond_to?(method_name, include_private = false) #:nodoc:
@tempfile.respond_to?(method_name, include_private) || super
end
+
+ private
+
+ def validate_kwargs(kwargs)
+ invalid_kwargs = kwargs.keys - ALLOWED_KWARGS
+ raise ArgumentError, "unknown keyword(s): #{invalid_kwargs.join(', ')}" if invalid_kwargs.any?
+ end
end